Merge pull request #305 from bxcodec/fixClickHouse

hotfix(dependency): resolve clickhouse dependency breaking changes

.dockerignore

@@ -0,0 +1,13 @@
+# Project
+FAQ.md
+README.md
+LICENSE
+Makefile
+.gitignore
+.travis.yml
+CONTRIBUTING.md
+MIGRATIONS.md
+docker-deploy.sh
+
+# Golang
+testing

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,39 @@
+---
+name: Bug report
+about: Create a report to help us improve
+
+---
+
+**Describe the Bug**
+A clear and concise description of what the bug is.
+
+**Steps to Reproduce**
+Steps to reproduce the behavior:
+1. My migrations look like '...'
+2. I ran migrate with the following options '....'
+3. See error
+
+**Expected Behavior**
+A clear and concise description of what you expected to happen.
+
+**Migrate Version**
+e.g. v3.4.0
+Obtained by running: `migrate -version`
+
+**Loaded Source Drivers**
+e.g. s3, github, go-bindata, gcs, file
+Obtained by running: `migrate -help`
+
+**Loaded Database Drivers**
+e.g. spanner, stub, clickhouse, cockroachdb, crdb-postgres, postgres, postgresql, redshift, cassandra, cockroach, mysql
+Obtained by running: `migrate -help`
+
+**Go Version**
+e.g. go version go1.11 linux/amd64
+Obtained by running: `go version`
+
+**Stacktrace**
+Please provide if available
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,17 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.gitignore

@@ -4,3 +4,5 @@ cli/cli
 cli/migrate
 .coverage
 .godoc.pid
+vendor/
+.vscode/

.golangci.yml

@@ -0,0 +1,27 @@
+run:
+  # timeout for analysis, e.g. 30s, 5m, default is 1m
+  timeout: 2m
+linters:
+  enable:
+    #- golint
+    - interfacer
+    - unconvert
+    #- dupl
+    - goconst
+    - gofmt
+    - misspell
+    - maligned
+    - unparam
+    - nakedret
+    - prealloc
+    #- gosec
+linters-settings:
+  misspell:
+    locale: US
+issues:
+  max-same: 0
+  max-per-linter: 0
+  exclude-use-default: false
+  exclude:
+    # gosec: Duplicated errcheck checks
+    - G104

.travis.yml

@@ -1,44 +1,64 @@
 language: go
 sudo: required
 
-go:
-  - 1.7
-  - 1.8
+matrix:
+  allow_failures:
+    - go: master
+  include:
+    # Supported versions of Go: https://golang.org/dl/
+    - go: "1.12.x"
+    - go: "1.13.x"
+    - go: master
+
+go_import_path: github.com/golang-migrate/migrate
 
 env:
-  - MIGRATE_TEST_CONTAINER_BOOT_DELAY=10
+  global:
+    - GO111MODULE=on
+    - MIGRATE_TEST_CONTAINER_BOOT_TIMEOUT=60
+    - DOCKER_USERNAME=golangmigrate
+    -   secure: "oSOznzUrgr5h45qW4PONkREpisPAt40tnM+KFWtS/Ggu5UI2Ie0CmyYXWuBjbt7B97a4yN9Qzmn8FxJHJ7kk+ABOi3muhkxeIhr6esXbzHhX/Jhv0mj1xkzX7KoVN9oHBz3cOI/QeRyEAO68xjDHNE2kby4RTT9VBt6TQUakKVkqI5qkqLBTADepCjVC+9XhxVxUNyeWKU8ormaUfJBjoNVoDlwXekUPnJenfmfZqXxUInvBCfUyp7Pq+kurBORmg4yc6qOlRYuK67Xw+i5xpjbZouNlXPk0rq7pPy5zjhmZQ3kImoFPvNMeKViDcI6kSIJKtjdhms9/g/6MgXS9HlL5kFy8tYKbsyiHnHB1BsvaLAKXctbUZFDPstgMPADfnad2kZXPrNqIhfWKZrGRWidawCYJ1sKKwYxLMKrtA0umqgMoL90MmBOELhuGmvMV0cFJB+zo+K2YWjEiMGd8xRb5mC5aAy0ZcCehO46jGtpr217EJmMF8Ywr7cFqM2Shg5U2jev9qUpYiXwmPnJKDuoT2ZHuHmPgFIkYiWC5yeJnnmG5bed1sKBp93AFrJX+1Rx5oC4BpNegewmBZKpOSwls/D1uMAeQK3dPmQHLsT6o2VBLfeDGr+zY0R85ywwPZCv00vGol02zYoTqN7eFqr6Qhjr/qx5K1nnxJdFK3Ts="
 
-# TODO: https://docs.docker.com/engine/installation/linux/ubuntu/    
-# pre-provision with travis docker setup and pin down docker version in install step
 services:
-  - docker 
+  - docker
+
+cache:
+  directories:
+    - $GOPATH/pkg
+
+
+before_install:
+  # Update docker to latest version: https://docs.travis-ci.com/user/docker/#installing-a-newer-docker-version
+  - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+  - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+  - sudo apt-get update
+  - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
+  # Install golangci-lint
+  - curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin v1.20.0
+  - echo "TRAVIS_GO_VERSION=${TRAVIS_GO_VERSION}"
 
 install:
-  - make deps
-  - (cd $GOPATH/src/github.com/docker/docker && git fetch --all --tags --prune && git checkout v1.13.0)
-  - sudo apt-get update && sudo apt-get install docker-engine=1.13.0*
-  - go get github.com/mattn/goveralls 
+  - go get github.com/mattn/goveralls
 
 script:
-  - make test
+  - golangci-lint run
+  - make test COVERAGE_DIR=/tmp/coverage
 
 after_success:
-  - goveralls -service=travis-ci -coverprofile .coverage/combined.txt
+  - goveralls -service=travis-ci -coverprofile /tmp/coverage/combined.txt
   - make list-external-deps > dependency_tree.txt && cat dependency_tree.txt
-
-before_deploy:
   - make build-cli
-  - gem install --no-ri --no-rdoc fpm 
-  - fpm -s dir -t deb -n migrate -v "$(git describe --tags 2>/dev/null | cut -c 2-)" --license MIT -m matthias.kadenbach@gmail.com --url https://github.com/mattes/migrate --description='Database migrations' -a amd64 -p migrate.$(git describe --tags 2>/dev/null | cut -c 2-).deb --deb-no-default-config-files -f -C cli/build migrate.linux-amd64=/usr/bin/migrate
+  - gem install --no-document fpm
+  - fpm -s dir -t deb -n migrate -v "$(git describe --tags 2>/dev/null | cut -c 2-)" --license MIT -m dhui@users.noreply.github.com --url https://github.com/golang-migrate/migrate --description='Database migrations' -a amd64 -p migrate.$(git describe --tags 2>/dev/null | cut -c 2-).deb --deb-no-default-config-files -f -C cli/build migrate.linux-amd64=/usr/local/bin/migrate
 
 deploy:
   - provider: releases
     api_key:
-      secure: EFow50BI448HVb/uQ1Kk2Kq0xzmwIYq3V67YyymXIuqSCodvXEsMiBPUoLrxEknpPEIc67LEQTNdfHBgvyHk6oRINWAfie+7pr5tKrpOTF9ghyxoN1PlO8WKQCqwCvGMBCnc5ur5rvzp0bqfpV2rs5q9/nngy3kBuEvs12V7iho=
+      secure: hWH1HLPpzpfA8pXQ93T1qKQVFSpQp0as/JLQ7D91jHuJ8p+RxVeqblDrR6HQY/95R/nyiE9GJmvUolSuw5h449LSrGxPtVWhdh6EnkxlQHlen5XeMhVjRjFV0sE9qGe8v7uAkiTfRO61ktTWHrEAvw5qpyqnNISodmZS78XIasPODQbNlzwINhWhDTHIjXGb4FpizYaL3OGCanrxfR9fQyCaqKGGBjRq3Mfq8U6Yd4mApmsE+uJxgaZV8K5zBqpkSzQRWhcVGNL5DuLsU3gfSJOo7kZeA2G71SHffH577dBoqtCZ4VFv169CoUZehLWCb+7XKJZmHXVujCURATSySLGUOPc6EoLFAn3YtsCA04mS4bZVo5FZPWVwfhjmkhtDR4f6wscKp7r1HsFHSOgm59QfETQdrn4MnZ44H2Jd39axqndn5DvK9EcZVjPHynOPnueXP2u6mTuUgh2VyyWBCDO3CNo0fGlo7VJI69IkIWNSD87K9cHZWYMClyKZkUzS+PmRAhHRYbVd+9ZjKOmnU36kUHNDG/ft1D4ogsY+rhVtXB4lgWDM5adri+EIScYdYnB1/pQexLBigcJY9uE7nQTR0U6QgVNYvun7uRNs40E0c4voSfmPdFO0FlOD2y1oQhnaXfWLbu9nMcTcs4RFGrcC7NzkUN4/WjG8s285V6w=
     skip_cleanup: true
     on:
-      go: 1.8
-      repo: mattes/migrate
+      go: "1.13.x"
+      repo: golang-migrate/migrate
       tags: true
     file:
       - cli/build/migrate.linux-amd64.tar.gz
@@ -48,14 +68,68 @@ deploy:
       - dependency_tree.txt
   - provider: packagecloud
     repository: migrate
-    username: mattes
+    username: golang-migrate
     token:
-      secure: RiHJ/+J9DvXUah/APYdWySWZ5uOOISYJ0wS7xddc7/BNStRVjzFzvJ9zmb67RkyZZrvGuVjPiL4T8mtDyCJCj47RmU/56wPdEHbar/FjsiUCgwvR19RlulkgbV4okBCePbwzMw6HNHRp14TzfQCPtnN4kef0lOI4gZJkImN7rtQ=
+      secure: aICwu3gJ1sJ1QVCD3elpg+Jxzt4P+Zj1uoh5f0sOwnjDNIZ4FwUT1cMrWloP8P2KD0iyCOawuZER27o/kQ21oX2OxHvQbYPReA2znLm7lHzCmypAAOHPxpgnQ4rMGHHJXd+OsxtdclGs67c+EbdBfoRRbK400Qz/vjPJEDeH4mh02ZHC2nw4Nk/wV4jjBIkIt9dGEx6NgOA17FCMa3MaPHlHeFIzU7IfTlDHbS0mCCYbg/wafWBWcbGqtZLWAYtJDmfjrAStmDLdAX5J5PsB7taGSGPZHmPmpGoVgrKt/tb9Xz1rFBGslTpGROOiO4CiMAvkEKFn8mxrBGjfSBqp7Dp3eeSalKXB1DJAbEXx2sEbMcvmnoR9o43meaAn+ZRts8lRL8S/skBloe6Nk8bx3NlJCGB9WPK1G56b7c/fZnJxQbrCw6hxDfbZwm8S2YPviFTo/z1BfZDhRsL74reKsN2kgnGo2W/k38vvzIpsssQ9DHN1b0TLCxolCNPtQ7oHcQ1ohcjP2UgYXk0FhqDoL+9LQva/DU4N9sKH0UbAaqsMVSErLeG8A4aauuFcVrWRBaDYyTag4dQqzTulEy7iru2kDDIBgSQ1gMW/yoBOIPK4oi6MtbTf1X39fzXFLS1cDd3LW61yAu3YrbjAetpfx2frIvrRAiL9TxWA1gnrs5o=
     dist: ubuntu/xenial
     package_glob: '*.deb'
     skip_cleanup: true
     on:
-      go: 1.8
-      repo: mattes/migrate
+      go: "1.13.x"
+      repo: golang-migrate/migrate
+      tags: true
+  - provider: packagecloud
+    repository: migrate
+    username: golang-migrate
+    token:
+      secure: aICwu3gJ1sJ1QVCD3elpg+Jxzt4P+Zj1uoh5f0sOwnjDNIZ4FwUT1cMrWloP8P2KD0iyCOawuZER27o/kQ21oX2OxHvQbYPReA2znLm7lHzCmypAAOHPxpgnQ4rMGHHJXd+OsxtdclGs67c+EbdBfoRRbK400Qz/vjPJEDeH4mh02ZHC2nw4Nk/wV4jjBIkIt9dGEx6NgOA17FCMa3MaPHlHeFIzU7IfTlDHbS0mCCYbg/wafWBWcbGqtZLWAYtJDmfjrAStmDLdAX5J5PsB7taGSGPZHmPmpGoVgrKt/tb9Xz1rFBGslTpGROOiO4CiMAvkEKFn8mxrBGjfSBqp7Dp3eeSalKXB1DJAbEXx2sEbMcvmnoR9o43meaAn+ZRts8lRL8S/skBloe6Nk8bx3NlJCGB9WPK1G56b7c/fZnJxQbrCw6hxDfbZwm8S2YPviFTo/z1BfZDhRsL74reKsN2kgnGo2W/k38vvzIpsssQ9DHN1b0TLCxolCNPtQ7oHcQ1ohcjP2UgYXk0FhqDoL+9LQva/DU4N9sKH0UbAaqsMVSErLeG8A4aauuFcVrWRBaDYyTag4dQqzTulEy7iru2kDDIBgSQ1gMW/yoBOIPK4oi6MtbTf1X39fzXFLS1cDd3LW61yAu3YrbjAetpfx2frIvrRAiL9TxWA1gnrs5o=
+    dist: ubuntu/bionic
+    package_glob: '*.deb'
+    skip_cleanup: true
+    on:
+      go: "1.13.x"
+      repo: golang-migrate/migrate
+      tags: true
+  - provider: packagecloud
+    repository: migrate
+    username: golang-migrate
+    token:
+      secure: aICwu3gJ1sJ1QVCD3elpg+Jxzt4P+Zj1uoh5f0sOwnjDNIZ4FwUT1cMrWloP8P2KD0iyCOawuZER27o/kQ21oX2OxHvQbYPReA2znLm7lHzCmypAAOHPxpgnQ4rMGHHJXd+OsxtdclGs67c+EbdBfoRRbK400Qz/vjPJEDeH4mh02ZHC2nw4Nk/wV4jjBIkIt9dGEx6NgOA17FCMa3MaPHlHeFIzU7IfTlDHbS0mCCYbg/wafWBWcbGqtZLWAYtJDmfjrAStmDLdAX5J5PsB7taGSGPZHmPmpGoVgrKt/tb9Xz1rFBGslTpGROOiO4CiMAvkEKFn8mxrBGjfSBqp7Dp3eeSalKXB1DJAbEXx2sEbMcvmnoR9o43meaAn+ZRts8lRL8S/skBloe6Nk8bx3NlJCGB9WPK1G56b7c/fZnJxQbrCw6hxDfbZwm8S2YPviFTo/z1BfZDhRsL74reKsN2kgnGo2W/k38vvzIpsssQ9DHN1b0TLCxolCNPtQ7oHcQ1ohcjP2UgYXk0FhqDoL+9LQva/DU4N9sKH0UbAaqsMVSErLeG8A4aauuFcVrWRBaDYyTag4dQqzTulEy7iru2kDDIBgSQ1gMW/yoBOIPK4oi6MtbTf1X39fzXFLS1cDd3LW61yAu3YrbjAetpfx2frIvrRAiL9TxWA1gnrs5o=
+    dist: ubuntu/cosmic
+    package_glob: '*.deb'
+    skip_cleanup: true
+    on:
+      go: "1.13.x"
+      repo: golang-migrate/migrate
+      tags: true
+  - provider: packagecloud
+    repository: migrate
+    username: golang-migrate
+    token:
+      secure: aICwu3gJ1sJ1QVCD3elpg+Jxzt4P+Zj1uoh5f0sOwnjDNIZ4FwUT1cMrWloP8P2KD0iyCOawuZER27o/kQ21oX2OxHvQbYPReA2znLm7lHzCmypAAOHPxpgnQ4rMGHHJXd+OsxtdclGs67c+EbdBfoRRbK400Qz/vjPJEDeH4mh02ZHC2nw4Nk/wV4jjBIkIt9dGEx6NgOA17FCMa3MaPHlHeFIzU7IfTlDHbS0mCCYbg/wafWBWcbGqtZLWAYtJDmfjrAStmDLdAX5J5PsB7taGSGPZHmPmpGoVgrKt/tb9Xz1rFBGslTpGROOiO4CiMAvkEKFn8mxrBGjfSBqp7Dp3eeSalKXB1DJAbEXx2sEbMcvmnoR9o43meaAn+ZRts8lRL8S/skBloe6Nk8bx3NlJCGB9WPK1G56b7c/fZnJxQbrCw6hxDfbZwm8S2YPviFTo/z1BfZDhRsL74reKsN2kgnGo2W/k38vvzIpsssQ9DHN1b0TLCxolCNPtQ7oHcQ1ohcjP2UgYXk0FhqDoL+9LQva/DU4N9sKH0UbAaqsMVSErLeG8A4aauuFcVrWRBaDYyTag4dQqzTulEy7iru2kDDIBgSQ1gMW/yoBOIPK4oi6MtbTf1X39fzXFLS1cDd3LW61yAu3YrbjAetpfx2frIvrRAiL9TxWA1gnrs5o=
+    dist: debian/stretch
+    package_glob: '*.deb'
+    skip_cleanup: true
+    on:
+      go: "1.13.x"
+      repo: golang-migrate/migrate
+      tags: true
+  - provider: packagecloud
+    repository: migrate
+    username: golang-migrate
+    token:
+      secure: aICwu3gJ1sJ1QVCD3elpg+Jxzt4P+Zj1uoh5f0sOwnjDNIZ4FwUT1cMrWloP8P2KD0iyCOawuZER27o/kQ21oX2OxHvQbYPReA2znLm7lHzCmypAAOHPxpgnQ4rMGHHJXd+OsxtdclGs67c+EbdBfoRRbK400Qz/vjPJEDeH4mh02ZHC2nw4Nk/wV4jjBIkIt9dGEx6NgOA17FCMa3MaPHlHeFIzU7IfTlDHbS0mCCYbg/wafWBWcbGqtZLWAYtJDmfjrAStmDLdAX5J5PsB7taGSGPZHmPmpGoVgrKt/tb9Xz1rFBGslTpGROOiO4CiMAvkEKFn8mxrBGjfSBqp7Dp3eeSalKXB1DJAbEXx2sEbMcvmnoR9o43meaAn+ZRts8lRL8S/skBloe6Nk8bx3NlJCGB9WPK1G56b7c/fZnJxQbrCw6hxDfbZwm8S2YPviFTo/z1BfZDhRsL74reKsN2kgnGo2W/k38vvzIpsssQ9DHN1b0TLCxolCNPtQ7oHcQ1ohcjP2UgYXk0FhqDoL+9LQva/DU4N9sKH0UbAaqsMVSErLeG8A4aauuFcVrWRBaDYyTag4dQqzTulEy7iru2kDDIBgSQ1gMW/yoBOIPK4oi6MtbTf1X39fzXFLS1cDd3LW61yAu3YrbjAetpfx2frIvrRAiL9TxWA1gnrs5o=
+    dist: debian/buster
+    package_glob: '*.deb'
+    skip_cleanup: true
+    on:
+      go: "1.13.x"
+      repo: golang-migrate/migrate
+      tags: true
+  - provider: script
+    script: ./docker-deploy.sh
+    skip_cleanup: true
+    on:
+      go: "1.13.x"
+      repo: golang-migrate/migrate
       tags: true
-

CONTRIBUTING.md

@@ -2,21 +2,23 @@
 
   1. Make sure you have a running Docker daemon
      (Install for [MacOS](https://docs.docker.com/docker-for-mac/))
-  2. Fork this repo and `git clone` somewhere to `$GOPATH/src/github.com/%you%/migrate`
-  3. `make rewrite-import-paths` to update imports to your local fork
-  4. Confirm tests are working: `make test-short`
-  5. Write awesome code ...
-  6. `make test` to run all tests against all database versions
-  7. `make restore-import-paths` to restore import paths
-  8. Push code and open Pull Request
+  1. Use a version of Go that supports [modules](https://golang.org/cmd/go/#hdr-Modules__module_versions__and_more) (e.g. Go 1.11+)
+  1. Fork this repo and `git clone` somewhere to `$GOPATH/src/github.com/golang-migrate/migrate`
+      * Ensure that [Go modules are enabled](https://golang.org/cmd/go/#hdr-Preliminary_module_support) (e.g. your repo path or the `GO111MODULE` environment variable are set correctly)
+  1. Install [golangci-lint](https://github.com/golangci/golangci-lint#install)
+  1. Run the linter: `golangci-lint run`
+  1. Confirm tests are working: `make test-short`
+  1. Write awesome code ...
+  1. `make test` to run all tests against all database versions
+  1. Push code and open Pull Request
  
 Some more helpful commands:
 
-  * You can specify which database/ source tests to run:  
-    `make test-short SOURCE='file go-bindata' DATABASE='postgres cassandra'`
-  * After `make test`, run `make html-coverage` which opens a shiny test coverage overview.  
-  * Missing imports? `make deps`
+  * You can specify which database/ source tests to run:
+    `make test-short SOURCE='file go_bindata' DATABASE='postgres cassandra'`
+  * After `make test`, run `make html-coverage` which opens a shiny test coverage overview.
   * `make build-cli` builds the CLI in directory `cli/build/`.
   * `make list-external-deps` lists all external dependencies for each package
-  * `make docs && make open-docs` opens godoc in your browser, `make kill-docs` kills the godoc server.  
-    Repeatedly call `make docs` to refresh the server.  
+  * `make docs && make open-docs` opens godoc in your browser, `make kill-docs` kills the godoc server.
+    Repeatedly call `make docs` to refresh the server.
+  * Set the `DOCKER_API_VERSION` environment variable to the latest supported version if you get errors regarding the docker client API version being too new.

Dockerfile

@@ -0,0 +1,23 @@
+FROM golang:1.12-alpine3.10 AS downloader
+ARG VERSION
+
+RUN apk add --no-cache git gcc musl-dev
+
+WORKDIR /go/src/github.com/golang-migrate/migrate
+
+COPY . ./
+
+ENV GO111MODULE=on
+ENV DATABASES="postgres mysql redshift cassandra spanner cockroachdb clickhouse mongodb sqlserver firebird"
+ENV SOURCES="file go_bindata github github_ee aws_s3 google_cloud_storage godoc_vfs gitlab"
+
+RUN go build -a -o build/migrate.linux-386 -ldflags="-s -w -X main.Version=${VERSION}" -tags "$DATABASES $SOURCES" ./cmd/migrate
+
+FROM alpine:3.10
+
+RUN apk add --no-cache ca-certificates
+
+COPY --from=downloader /go/src/github.com/golang-migrate/migrate/build/migrate.linux-386 /migrate
+
+ENTRYPOINT ["/migrate"]
+CMD ["--help"]

FAQ.md

@@ -1,67 +1,76 @@
-# FAQ
-
-#### How is the code base structured?
-  ```
-  /          package migrate (the heart of everything)
-  /cli       the CLI wrapper
-  /database  database driver and sub directories have the actual driver implementations
-  /source    source driver and sub directories have the actual driver implementations
-  ```
-
-#### Why is there no `source/driver.go:Last()`?
-  It's not needed. And unless the source has a "native" way to read a directory in reversed order,
-  it might be expensive to do a full directory scan in order to get the last element.
-
-#### What is a NilMigration? NilVersion?
-  NilMigration defines a migration without a body. NilVersion is defined as const -1.
-
-#### What is the difference between uint(version) and int(targetVersion)?
-  version refers to an existing migration version coming from a source and therefor can never be negative. 
-  targetVersion can either be a version OR represent a NilVersion, which equals -1.    
-
-#### What's the difference between Next/Previous and Up/Down?
-  ```
-  1_first_migration.up           next ->  2_second_migration.up      ...
-  1_first_migration.down  <- previous     2_second_migration.down    ...
-  ```
- 
-#### Why two separate files (up and down) for a migration?
-  It makes all of our lives easier. No new markup/syntax to learn for users 
-  and existing database utility tools continue to work as expected.
-    
-#### How many migrations can migrate handle?
-  Whatever the maximum positive signed integer value is for your platform. 
-  For 32bit it would be 2,147,483,647 migrations. Migrate only keeps references to 
-  the currently run and pre-fetched migrations in memory. Please note that some 
-  source drivers need to do build a full "directory" tree first, which puts some
-  heat on the memory consumption. 
-    
-#### Are the table tests in migrate_test.go bloated?
-  Yes and no. There are duplicate test cases for sure but they don't hurt here. In fact
-  the tests are very visual now and might help new users understand expected behaviors quickly.
-  Migrate from version x to y and y is the last migration? Just check out the test for 
-  that particular case and know what's going on instantly.
-    
-#### What is Docker being used for?
-  Only for testing. See [testing/docker.go](testing/docker.go)
-    
-#### Why not just use docker-compose?
-  It doesn't give us enough runtime control for testing. We want to be able to bring up containers fast
-  and whenever we want, not just once at the beginning of all tests.
-    
-#### Can I maintain my driver in my own repository?
-  Yes, technically thats possible. We want to encourage you to contribute your driver to this respository though.
-  The driver's functionality is dictated by migrate's interfaces. That means there should really
-  just be one driver for a database/ source. We want to prevent a future where several drivers doing the exact same thing,
-  just implemented a bit differently, co-exist somewhere on Github. If users have to do research first to find the
-  "best" available driver for a database in order to get started, we would have failed as an open source community.
-
-#### Can I mix multiple sources during a batch of migrations?
-  No.
-
-#### What does "dirty" database mean?
-  Before a migration runs, each database sets a dirty flag. Execution stops if a migration fails and the dirty state persists,
-  which prevents attempts to run more migrations on top of a failed migration. You need to manually fix the error
-  and then "force" the expected version.
-
-  
+# FAQ
+
+#### How is the code base structured?
+  ```
+  /          package migrate (the heart of everything)
+  /cli       the CLI wrapper
+  /database  database driver and sub directories have the actual driver implementations
+  /source    source driver and sub directories have the actual driver implementations
+  ```
+
+#### Why is there no `source/driver.go:Last()`?
+  It's not needed. And unless the source has a "native" way to read a directory in reversed order,
+  it might be expensive to do a full directory scan in order to get the last element.
+
+#### What is a NilMigration? NilVersion?
+  NilMigration defines a migration without a body. NilVersion is defined as const -1.
+
+#### What is the difference between uint(version) and int(targetVersion)?
+  version refers to an existing migration version coming from a source and therefor can never be negative.
+  targetVersion can either be a version OR represent a NilVersion, which equals -1.
+
+#### What's the difference between Next/Previous and Up/Down?
+  ```
+  1_first_migration.up.extension           next ->  2_second_migration.up.extension      ...
+  1_first_migration.down.extension  <- previous     2_second_migration.down.extension    ...
+  ```
+
+#### Why two separate files (up and down) for a migration?
+  It makes all of our lives easier. No new markup/syntax to learn for users
+  and existing database utility tools continue to work as expected.
+
+#### How many migrations can migrate handle?
+  Whatever the maximum positive signed integer value is for your platform.
+  For 32bit it would be 2,147,483,647 migrations. Migrate only keeps references to
+  the currently run and pre-fetched migrations in memory. Please note that some
+  source drivers need to do build a full "directory" tree first, which puts some
+  heat on the memory consumption.
+
+#### Are the table tests in migrate_test.go bloated?
+  Yes and no. There are duplicate test cases for sure but they don't hurt here. In fact
+  the tests are very visual now and might help new users understand expected behaviors quickly.
+  Migrate from version x to y and y is the last migration? Just check out the test for
+  that particular case and know what's going on instantly.
+
+#### What is Docker being used for?
+  Only for testing. See [testing/docker.go](testing/docker.go)
+
+#### Why not just use docker-compose?
+  It doesn't give us enough runtime control for testing. We want to be able to bring up containers fast
+  and whenever we want, not just once at the beginning of all tests.
+
+#### Can I maintain my driver in my own repository?
+  Yes, technically thats possible. We want to encourage you to contribute your driver to this respository though.
+  The driver's functionality is dictated by migrate's interfaces. That means there should really
+  just be one driver for a database/ source. We want to prevent a future where several drivers doing the exact same thing,
+  just implemented a bit differently, co-exist somewhere on Github. If users have to do research first to find the
+  "best" available driver for a database in order to get started, we would have failed as an open source community.
+
+#### Can I mix multiple sources during a batch of migrations?
+  No.
+
+#### What does "dirty" database mean?
+  Before a migration runs, each database sets a dirty flag. Execution stops if a migration fails and the dirty state persists,
+  which prevents attempts to run more migrations on top of a failed migration. You need to manually fix the error
+  and then "force" the expected version.
+
+#### What happens if two programs try and update the database at the same time?
+Database-specific locking features are used by *some* database drivers to prevent multiple instances of migrate from running migrations at the same time
+  the same database at the same time. For example, the MySQL driver uses the `GET_LOCK` function, while the Postgres driver uses
+  the `pg_advisory_lock` function.
+
+#### Do I need to create a table for tracking migration version used?
+No, it is done automatically.
+
+#### Can I use migrate with a non-Go project?
+Yes, you can use the migrate CLI in a non-Go project, but there are probably other libraries/frameworks available that offer better test and deploy integrations in that language/framework.

GETTING_STARTED.md

@@ -0,0 +1,43 @@
+# Getting started
+Before you start, you should understand the concept of forward/up and reverse/down database migrations.
+
+Configure a database for your application. Make sure that your database driver is supported [here](README.md#databases)
+
+## Create migrations
+Create some migrations using migrate CLI. Here is an example:
+```
+migrate create -ext sql -dir db/migrations -seq create_users_table
+```
+Once you create your files, you should fill them.
+
+**IMPORTANT:** In a project developed by more than one person there is a chance of migrations inconsistency - e.g. two developers can create conflicting migrations, and the developer that created his migration later gets it merged to the repository first.
+Developers and Teams should keep an eye on such cases (especially during code review).
+[Here](https://github.com/golang-migrate/migrate/issues/179#issuecomment-475821264) is the issue summary if you would like to read more.
+
+Consider making your migrations idempotent - we can run the same sql code twice in a row with the same result. This makes our migrations more robust. On the other hand, it causes slightly less control over database schema - e.g. let's say you forgot to drop the table in down migration. You run down migration - the table is still there. When you run up migration again - `CREATE TABLE` would return an error, helping you find an issue in down migration, while `CREATE TABLE IF NOT EXISTS` would not. Use those conditions wisely.
+
+In case you would like to run several commands/queries in one migration, you should wrap them in a transaction (if your database supports it).
+This way if one of commands fails, our database will remain unchanged.
+
+## Run migrations
+Run your migrations through the CLI or your app and check if they applied expected changes.
+Just to give you an idea:
+```
+migrate -database YOUR_DATBASE_URL -path PATH_TO_YOUR_MIGRATIONS up
+```
+
+Just add the code to your app and you're ready to go!
+
+Before commiting your migrations you should run your migrations up, down, and then up again to see if migrations are working properly both ways.
+(e.g. if you created a table in a migration but reverse migration did not delete it, you will encounter an error when running the forward migration again)
+It's also worth checking your migrations in a separate, containerized environment. You can find some tools in the end of this document.
+
+**IMPORTANT:** If you would like to run multiple instances of your app on different machines be sure to use a database that supports locking when running migrations. Otherwise you may encounter issues.
+
+## Further reading:
+- [PostgreSQL tutorial](database/postgres/TUTORIAL.md)
+- [Best practices](MIGRATIONS.md)
+- [FAQ](FAQ.md)
+- Tools for testing your migrations in a container:
+	- https://github.com/dhui/dktest
+	- https://github.com/ory/dockertest

LICENSE

@@ -1,9 +1,14 @@
 The MIT License (MIT)
 
+Original Work
 Copyright (c) 2016 Matthias Kadenbach
-
 https://github.com/mattes/migrate
 
+Modified Work
+Copyright (c) 2018 Dale Hui
+https://github.com/golang-migrate/migrate
+
+
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
 in the Software without restriction, including without limitation the rights

MIGRATIONS.md

@@ -1,5 +1,86 @@
-# Migrations
-
-## Best practices: How to write migrations.
-
-@TODO
+# Migrations
+
+## Migration Filename Format
+
+A single logical migration is represented as two separate migration files, one
+to migrate "up" to the specified version from the previous version, and a second
+to migrate back "down" to the previous version.  These migrations can be provided
+by any one of the supported [migration sources](./README.md#migration-sources).
+
+The ordering and direction of the migration files is determined by the filenames
+used for them.  `migrate` expects the filenames of migrations to have the format:
+
+    {version}_{title}.up.{extension}
+    {version}_{title}.down.{extension}
+
+The `title` of each migration is unused, and is only for readability.  Similarly,
+the `extension` of the migration files is not checked by the library, and should
+be an appropriate format for the database in use (`.sql` for SQL variants, for
+instance).
+
+Versions of migrations may be represented as any 64 bit unsigned integer.
+All migrations are applied upward in order of increasing version number, and
+downward by decreasing version number.
+
+Common versioning schemes include incrementing integers:
+
+    1_initialize_schema.down.sql
+    1_initialize_schema.up.sql
+    2_add_table.down.sql
+    2_add_table.up.sql
+    ...
+
+Or timestamps at an appropriate resolution:
+
+    1500360784_initialize_schema.down.sql
+    1500360784_initialize_schema.up.sql
+    1500445949_add_table.down.sql
+    1500445949_add_table.up.sql
+    ...
+
+But any scheme resulting in distinct, incrementing integers as versions is valid.
+
+It is suggested that the version number of corresponding `up` and `down` migration
+files be equivalent for clarity, but they are allowed to differ so long as the
+relative ordering of the migrations is preserved.
+
+The migration files are permitted to be "empty", in the event that a migration
+is a no-op or is irreversible. It is recommended to still include both migration
+files by making the whole migration file consist of a comment.
+If your database does not support comments, then deleting the migration file will also work.
+Note, an actual empty file (e.g. a 0 byte file) may cause issues with your database since migrate
+will attempt to run an empty query. In this case, deleting the migration file will also work.
+For the rational of this behavior see:
+[#244 (comment)](https://github.com/golang-migrate/migrate/issues/244#issuecomment-510758270)
+
+## Migration Content Format
+
+The format of the migration files themselves varies between database systems.
+Different databases have different semantics around schema changes and when and
+how they are allowed to occur
+(for instance, [if schema changes can occur within a transaction](https://wiki.postgresql.org/wiki/Transactional_DDL_in_PostgreSQL:_A_Competitive_Analysis)).
+
+As such, the `migrate` library has little to no checking around the format of
+migration sources.  The migration files are generally processed directly by the
+drivers as raw operations.
+
+## Reversibility of Migrations
+
+Best practice for writing schema migration is that all migrations should be
+reversible.  It should in theory be possible for run migrations down and back up
+through any and all versions with the state being fully cleaned and recreated
+by doing so.
+
+By adhering to this recommended practice, development and deployment of new code
+is cleaner and easier (cleaning database state for a new feature should be as
+easy as migrating down to a prior version, and back up to the latest).
+
+As opposed to some other migration libraries, `migrate` represents up and down
+migrations as separate files.  This prevents any non-standard file syntax from
+being introduced which may result in unintended behavior or errors, depending
+on what database is processing the file.
+
+While it is technically possible for an up or down migration to exist on its own
+without an equivalently versioned counterpart, it is strongly recommended to
+always include a down migration which cleans up the state of the corresponding
+up migration.

Makefile

@@ -1,15 +1,16 @@
-SOURCE ?= file go-bindata github aws-s3 google-cloud-storage
-DATABASE ?= postgres mysql redshift
+SOURCE ?= file go_bindata github github_ee aws_s3 google_cloud_storage godoc_vfs gitlab
+DATABASE ?= postgres mysql redshift cassandra spanner cockroachdb clickhouse mongodb sqlserver firebird
 VERSION ?= $(shell git describe --tags 2>/dev/null | cut -c 2-)
 TEST_FLAGS ?=
 REPO_OWNER ?= $(shell cd .. && basename "$$(pwd)")
+COVERAGE_DIR ?= .coverage
 
 
 build-cli: clean
 	-mkdir ./cli/build
-	cd ./cli && GOOS=linux GOARCH=amd64 go build -a -o build/migrate.linux-amd64 -ldflags='-X main.Version=$(VERSION)' -tags '$(DATABASE) $(SOURCE)' . 
-	cd ./cli && GOOS=darwin GOARCH=amd64 go build -a -o build/migrate.darwin-amd64 -ldflags='-X main.Version=$(VERSION)' -tags '$(DATABASE) $(SOURCE)' . 
-	cd ./cli && GOOS=windows GOARCH=amd64 go build -a -o build/migrate.windows-amd64.exe -ldflags='-X main.Version=$(VERSION)' -tags '$(DATABASE) $(SOURCE)' . 
+	cd ./cmd/migrate && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o ../../cli/build/migrate.linux-amd64 -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' .
+	cd ./cmd/migrate && CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -a -o ../../cli/build/migrate.darwin-amd64 -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' .
+	cd ./cmd/migrate && CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -a -o ../../cli/build/migrate.windows-amd64.exe -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' .
 	cd ./cli/build && find . -name 'migrate*' | xargs -I{} tar czf {}.tar.gz {}
 	cd ./cli/build && shasum -a 256 * > sha256sum.txt
 	cat ./cli/build/sha256sum.txt
@@ -24,28 +25,16 @@ test-short:
 
 
 test:
-	@-rm -r .coverage
-	@mkdir .coverage
-	make test-with-flags TEST_FLAGS='-v -race -covermode atomic -coverprofile .coverage/_$$(RAND).txt -bench=. -benchmem'
-	@echo 'mode: atomic' > .coverage/combined.txt
-	@cat .coverage/*.txt | grep -v 'mode: atomic' >> .coverage/combined.txt
+	@-rm -r $(COVERAGE_DIR)
+	@mkdir $(COVERAGE_DIR)
+	make test-with-flags TEST_FLAGS='-v -race -covermode atomic -coverprofile $$(COVERAGE_DIR)/combined.txt -bench=. -benchmem -timeout 20m'
 
 
 test-with-flags:
-	@echo SOURCE: $(SOURCE) 
+	@echo SOURCE: $(SOURCE)
 	@echo DATABASE: $(DATABASE)
 
-	@go test $(TEST_FLAGS) .
-	@go test $(TEST_FLAGS) ./cli/...
-	@go test $(TEST_FLAGS) ./testing/...
-
-	@echo -n '$(SOURCE)' | tr -s ' ' '\n' | xargs -I{} go test $(TEST_FLAGS) ./source/{}
-	@go test $(TEST_FLAGS) ./source/testing/...
-	@go test $(TEST_FLAGS) ./source/stub/...
-
-	@echo -n '$(DATABASE)' | tr -s ' ' '\n' | xargs -I{} go test $(TEST_FLAGS) ./database/{}
-	@go test $(TEST_FLAGS) ./database/testing/...
-	@go test $(TEST_FLAGS) ./database/stub/...
+	@go test $(TEST_FLAGS) ./...
 
 
 kill-orphaned-docker-containers:
@@ -53,14 +42,7 @@ kill-orphaned-docker-containers:
 
 
 html-coverage:
-	go tool cover -html=.coverage/combined.txt
-
-
-deps:
-	-go get -v -u ./... 
-	-go test -v -i ./...
-	# TODO: why is this not being fetched with the command above?
-	-go get -u github.com/fsouza/fake-gcs-server/fakestorage
+	go tool cover -html=$(COVERAGE_DIR)/combined.txt
 
 
 list-external-deps:
@@ -89,7 +71,7 @@ rewrite-import-paths:
 docs:
 	-make kill-docs
 	nohup godoc -play -http=127.0.0.1:6064 </dev/null >/dev/null 2>&1 & echo $$! > .godoc.pid
-	cat .godoc.pid  
+	cat .godoc.pid
 
 
 kill-docs:
@@ -114,7 +96,7 @@ define external_deps
 endef
 
 
-.PHONY: build-cli clean test-short test test-with-flags deps html-coverage \
+.PHONY: build-cli clean test-short test test-with-flags html-coverage \
         restore-import-paths rewrite-import-paths list-external-deps release \
         docs kill-docs open-docs kill-orphaned-docker-containers
 

README.md

@@ -1,86 +1,114 @@
-[![Build Status](https://travis-ci.org/mattes/migrate.svg?branch=master)](https://travis-ci.org/mattes/migrate)
-[![GoDoc](https://godoc.org/github.com/mattes/migrate?status.svg)](https://godoc.org/github.com/mattes/migrate)
-[![Coverage Status](https://coveralls.io/repos/github/mattes/migrate/badge.svg?branch=v3.0-prev)](https://coveralls.io/github/mattes/migrate?branch=v3.0-prev)
-[![packagecloud.io](https://img.shields.io/badge/deb-packagecloud.io-844fec.svg)](https://packagecloud.io/mattes/migrate?filter=debs)
+[![Build Status](https://img.shields.io/travis/com/golang-migrate/migrate/master.svg)](https://travis-ci.com/golang-migrate/migrate)
+[![GoDoc](https://godoc.org/github.com/golang-migrate/migrate?status.svg)](https://godoc.org/github.com/golang-migrate/migrate)
+[![Coverage Status](https://img.shields.io/coveralls/github/golang-migrate/migrate/master.svg)](https://coveralls.io/github/golang-migrate/migrate?branch=master)
+[![packagecloud.io](https://img.shields.io/badge/deb-packagecloud.io-844fec.svg)](https://packagecloud.io/golang-migrate/migrate?filter=debs)
+[![Docker Pulls](https://img.shields.io/docker/pulls/migrate/migrate.svg)](https://hub.docker.com/r/migrate/migrate/)
+![Supported Go Versions](https://img.shields.io/badge/Go-1.12%2C%201.13-lightgrey.svg)
+[![GitHub Release](https://img.shields.io/github/release/golang-migrate/migrate.svg)](https://github.com/golang-migrate/migrate/releases)
+[![Go Report Card](https://goreportcard.com/badge/github.com/golang-migrate/migrate)](https://goreportcard.com/report/github.com/golang-migrate/migrate) 
 
 # migrate
 
 __Database migrations written in Go. Use as [CLI](#cli-usage) or import as [library](#use-in-your-go-project).__
 
- * Migrate reads migrations from [sources](#migration-sources)
+* Migrate reads migrations from [sources](#migration-sources)
    and applies them in correct order to a [database](#databases).
- * Drivers are "dumb", migrate glues everything together and makes sure the logic is bulletproof.  
+* Drivers are "dumb", migrate glues everything together and makes sure the logic is bulletproof.
    (Keeps the drivers lightweight, too.)
- * Database drivers don't assume things or try to correct user input. When in doubt, fail.
-
-
-Looking for [v1](https://github.com/mattes/migrate/tree/v1)?
+* Database drivers don't assume things or try to correct user input. When in doubt, fail.
 
+Forked from [mattes/migrate](https://github.com/mattes/migrate)
 
 ## Databases
 
 Database drivers run migrations. [Add a new database?](database/driver.go)
 
-  * [PostgreSQL](database/postgres)
-  * [Redshift](database/redshift)
-  * [Ql](database/ql)
-  * [Cassandra](database/cassandra) ([todo #164](https://github.com/mattes/migrate/issues/164))
-  * [SQLite](database/sqlite) ([todo #165](https://github.com/mattes/migrate/issues/165))
-  * [MySQL/ MariaDB](database/mysql)
-  * [Neo4j](database/neo4j) ([todo #167](https://github.com/mattes/migrate/issues/167))
-  * [MongoDB](database/mongodb) ([todo #169](https://github.com/mattes/migrate/issues/169))
-  * [CrateDB](database/crate) ([todo #170](https://github.com/mattes/migrate/issues/170))
-  * [Shell](database/shell) ([todo #171](https://github.com/mattes/migrate/issues/171))
-  * [Google Cloud Spanner](database/spanner) ([todo #172](https://github.com/mattes/migrate/issues/172))
+* [PostgreSQL](database/postgres)
+* [Redshift](database/redshift)
+* [Ql](database/ql)
+* [Cassandra](database/cassandra)
+* [SQLite](database/sqlite3) ([todo #165](https://github.com/mattes/migrate/issues/165))
+* [MySQL/ MariaDB](database/mysql)
+* [Neo4j](database/neo4j) ([todo #167](https://github.com/mattes/migrate/issues/167))
+* [MongoDB](database/mongodb)
+* [CrateDB](database/crate) ([todo #170](https://github.com/mattes/migrate/issues/170))
+* [Shell](database/shell) ([todo #171](https://github.com/mattes/migrate/issues/171))
+* [Google Cloud Spanner](database/spanner)
+* [CockroachDB](database/cockroachdb)
+* [ClickHouse](database/clickhouse)
+* [Firebird](database/firebird)
+* [MS SQL Server](database/sqlserver)
 
+### Database URLs
 
+Database connection strings are specified via URLs. The URL format is driver dependent but generally has the form: `dbdriver://username:password@host:port/dbname?option1=true&option2=false`
+
+Any [reserved URL characters](https://en.wikipedia.org/wiki/Percent-encoding#Percent-encoding_reserved_characters) need to be escaped. Note, the `%` character also [needs to be escaped](https://en.wikipedia.org/wiki/Percent-encoding#Percent-encoding_the_percent_character)
+
+Explicitly, the following characters need to be escaped:
+`!`, `#`, `$`, `%`, `&`, `'`, `(`, `)`, `*`, `+`, `,`, `/`, `:`, `;`, `=`, `?`, `@`, `[`, `]`
+
+It's easiest to always run the URL parts of your DB connection URL (e.g. username, password, etc) through an URL encoder. See the example Python snippets below:
+
+```bash
+$ python3 -c 'import urllib.parse; print(urllib.parse.quote(input("String to encode: "), ""))'
+String to encode: FAKEpassword!#$%&'()*+,/:;=?@[]
+FAKEpassword%21%23%24%25%26%27%28%29%2A%2B%2C%2F%3A%3B%3D%3F%40%5B%5D
+$ python2 -c 'import urllib; print urllib.quote(raw_input("String to encode: "), "")'
+String to encode: FAKEpassword!#$%&'()*+,/:;=?@[]
+FAKEpassword%21%23%24%25%26%27%28%29%2A%2B%2C%2F%3A%3B%3D%3F%40%5B%5D
+$
+```
 
 ## Migration Sources
 
 Source drivers read migrations from local or remote sources. [Add a new source?](source/driver.go)
 
-  * [Filesystem](source/file) - read from fileystem (always included)
-  * [Go-Bindata](source/go-bindata) - read from embedded binary data ([jteeuwen/go-bindata](https://github.com/jteeuwen/go-bindata))
-  * [Github](source/github) - read from remote Github repositories
-  * [AWS S3](source/aws-s3) - read from Amazon Web Services S3
-  * [Google Cloud Storage](source/google-cloud-storage) - read from Google Cloud Platform Storage
-
-
+* [Filesystem](source/file) - read from filesystem
+* [Go-Bindata](source/go_bindata) - read from embedded binary data ([jteeuwen/go-bindata](https://github.com/jteeuwen/go-bindata))
+* [Github](source/github) - read from remote Github repositories
+* [Github Enterprise](source/github_ee) - read from remote Github Enterprise repositories
+* [Gitlab](source/gitlab) - read from remote Gitlab repositories
+* [AWS S3](source/aws_s3) - read from Amazon Web Services S3
+* [Google Cloud Storage](source/google_cloud_storage) - read from Google Cloud Platform Storage
 
 ## CLI usage
 
-  * Simple wrapper around this library.
-  * Handles ctrl+c (SIGINT) gracefully.
-  * No config search paths, no config files, no magic ENV var injections.
+* Simple wrapper around this library.
+* Handles ctrl+c (SIGINT) gracefully.
+* No config search paths, no config files, no magic ENV var injections.
 
-__[CLI Documentation](cli)__
+__[CLI Documentation](cmd/migrate)__
 
-([brew todo #156](https://github.com/mattes/migrate/issues/156))
+### Basic usage
 
-```
-$ brew install migrate --with-postgres
-$ migrate -database postgres://localhost:5432/database up 2
+```bash
+$ migrate -source file://path/to/migrations -database postgres://localhost:5432/database up 2
 ```
 
+### Docker usage
+
+```bash
+$ docker run -v {{ migration dir }}:/migrations --network host migrate/migrate
+    -path=/migrations/ -database postgres://localhost:5432/database up 2
+```
 
 ## Use in your Go project
 
- * API is stable and frozen for this release (v3.x).
- * Package migrate has no external dependencies.
- * Only import the drivers you need.
-   (check [dependency_tree.txt](https://github.com/mattes/migrate/releases) for each driver)
- * To help prevent database corruptions, it supports graceful stops via `GracefulStop chan bool`.
- * Bring your own logger.
- * Uses `io.Reader` streams internally for low memory overhead.
- * Thread-safe and no goroutine leaks.
+* API is stable and frozen for this release (v3 & v4).
+* Uses [Go modules](https://golang.org/cmd/go/#hdr-Modules__module_versions__and_more) to manage dependencies.
+* To help prevent database corruptions, it supports graceful stops via `GracefulStop chan bool`.
+* Bring your own logger.
+* Uses `io.Reader` streams internally for low memory overhead.
+* Thread-safe and no goroutine leaks.
 
-__[Go Documentation](https://godoc.org/github.com/mattes/migrate)__
+__[Go Documentation](https://godoc.org/github.com/golang-migrate/migrate)__
 
 ```go
 import (
-    "github.com/mattes/migrate"
-    _ "github.com/mattes/migrate/database/postgres"
-    _ "github.com/mattes/migrate/source/github"
+    "github.com/golang-migrate/migrate/v4"
+    _ "github.com/golang-migrate/migrate/v4/database/postgres"
+    _ "github.com/golang-migrate/migrate/v4/source/github"
 )
 
 func main() {
@@ -97,9 +125,9 @@ Want to use an existing database client?
 import (
     "database/sql"
     _ "github.com/lib/pq"
-    "github.com/mattes/migrate"
-    "github.com/mattes/migrate/database/postgres"
-    _ "github.com/mattes/migrate/source/file"
+    "github.com/golang-migrate/migrate/v4"
+    "github.com/golang-migrate/migrate/v4/database/postgres"
+    _ "github.com/golang-migrate/migrate/v4/source/file"
 )
 
 func main() {
@@ -112,18 +140,34 @@ func main() {
 }
 ```
 
+## Getting started
+
+Go to [getting started](GETTING_STARTED.md)
+
+## Tutorials
+
+- [PostgreSQL](database/postgres/TUTORIAL.md)
+
+(more tutorials to come)
+
 ## Migration files
 
 Each migration has an up and down migration. [Why?](FAQ.md#why-two-separate-files-up-and-down-for-a-migration)
 
-```
+```bash
 1481574547_create_users_table.up.sql
 1481574547_create_users_table.down.sql
 ```
 
 [Best practices: How to write migrations.](MIGRATIONS.md)
 
+## Versions
 
+Version | Supported? | Import | Notes
+--------|------------|--------|------
+**master** | :white_check_mark: | `import "github.com/golang-migrate/migrate/v4"` | New features and bug fixes arrive here first |
+**v4** | :white_check_mark: | `import "github.com/golang-migrate/migrate/v4"` | Used for stable releases |
+**v3** | :x: | `import "github.com/golang-migrate/migrate"` (with package manager) or `import "gopkg.in/golang-migrate/migrate.v3"` (not recommended) | **DO NOT USE** - No longer supported |
 
 ## Development and Contributing
 
@@ -132,12 +176,6 @@ read the [development guide](CONTRIBUTING.md).
 
 Also have a look at the [FAQ](FAQ.md).
 
-
-
 ---
 
-__Alternatives__
-
-https://bitbucket.org/liamstask/goose, https://github.com/tanel/dbmigrate,  
-https://github.com/BurntSushi/migration, https://github.com/DavidHuie/gomigrate,  
-https://github.com/rubenv/sql-migrate
+Looking for alternatives? [https://awesome-go.com/#database](https://awesome-go.com/#database).

cli/README.md

@@ -1,105 +1,3 @@
-# migrate CLI
+# Deprecated
 
-## Installation
-
-#### With Go toolchain
-
-```
-$ go get -u -d github.com/mattes/migrate/cli
-$ go build -tags 'postgres' -o /usr/local/bin/migrate github.com/mattes/migrate/cli
-```
-
-#### MacOS
-
-([todo #156](https://github.com/mattes/migrate/issues/156))
-
-```
-$ brew install migrate --with-postgres
-```
-
-#### Linux (*.deb package)
-
-```
-$ curl -L https://packagecloud.io/mattes/migrate/gpgkey | apt-key add -
-$ echo "deb https://packagecloud.io/mattes/migrate/ubuntu/ xenial main" > /etc/apt/sources.list.d/migrate.list
-$ apt-get update
-$ apt-get install -y migrate
-```
-
-#### Download pre-build binary (Windows, MacOS, or Linux)
-
-[Release Downloads](https://github.com/mattes/migrate/releases)
-
-```
-$ curl -L https://github.com/mattes/migrate/releases/download/$version/migrate.$platform-amd64.tar.gz | tar xvz
-```
-
-
-
-## Usage
-
-```
-$ migrate -help
-Usage: migrate OPTIONS COMMAND [arg...]
-       migrate [ -version | -help ]
-
-Options:
-  -source          Location of the migrations (driver://url)
-  -path            Shorthand for -source=file://path 
-  -database        Run migrations against this database (driver://url)
-  -prefetch N      Number of migrations to load in advance before executing (default 10)
-  -lock-timeout N  Allow N seconds to acquire database lock (default 15)
-  -verbose         Print verbose logging
-  -version         Print version
-  -help            Print usage
-
-Commands:
-  goto V       Migrate to version V
-  up [N]       Apply all or N up migrations
-  down [N]     Apply all or N down migrations
-  drop         Drop everyting inside database
-  force V      Set version V but don't run migration (ignores dirty state)
-  version      Print current migration version
-```
-
-
-So let's say you want to run the first two migrations
-
-```
-$ migrate -database postgres://localhost:5432/database up 2
-```
-
-If your migrations are hosted on github
-
-```
-$ migrate -source github://mattes:personal-access-token@mattes/migrate_test \
-    -database postgres://localhost:5432/database down 2
-```
-
-The CLI will gracefully stop at a safe point when SIGINT (ctrl+c) is received.
-Send SIGKILL for immediate halt.
-
-
-
-## Reading CLI arguments from somewhere else
-
-##### ENV variables
-
-```
-$ migrate -database "$MY_MIGRATE_DATABASE"
-```
-
-##### JSON files
-
-Check out https://stedolan.github.io/jq/
-
-```
-$ migrate -database "$(cat config.json | jq '.database')"
-```
-
-##### YAML files
-
-````
-$ migrate -database "$(cat config/database.yml | ruby -ryaml -e "print YAML.load(STDIN.read)['database']")"
-$ migrate -database "$(cat config/database.yml | python -c 'import yaml,sys;print yaml.safe_load(sys.stdin)["database"]')"
-```
+Use [cmd/migrate](../cmd/migrate) instead

cli/build_aws-s3.go

@@ -1,7 +0,0 @@
-// +build aws-s3
-
-package main
-
-import (
-	_ "github.com/mattes/migrate/source/aws-s3"
-)

cli/build_github.go

@@ -1,7 +0,0 @@
-// +build github
-
-package main
-
-import (
-	_ "github.com/mattes/migrate/source/github"
-)

cli/build_go-bindata.go

@@ -1,7 +0,0 @@
-// +build go-bindata
-
-package main
-
-import (
-	_ "github.com/mattes/migrate/source/go-bindata"
-)

cli/build_google-cloud-storage.go

@@ -1,7 +0,0 @@
-// +build google-cloud-storage
-
-package main
-
-import (
-	_ "github.com/mattes/migrate/source/google-cloud-storage"
-)

cli/build_mysql.go

@@ -1,7 +0,0 @@
-// +build mysql
-
-package main
-
-import (
-	_ "github.com/mattes/migrate/database/mysql"
-)

cli/build_postgres.go

@@ -1,7 +0,0 @@
-// +build postgres
-
-package main
-
-import (
-	_ "github.com/mattes/migrate/database/postgres"
-)

cli/build_ql.go

@@ -1,7 +0,0 @@
-// +build ql
-
-package main
-
-import (
-	_ "github.com/mattes/migrate/database/ql"
-)

cli/build_redshift.go

@@ -1,7 +0,0 @@
-// +build redshift
-
-package main
-
-import (
-	_ "github.com/mattes/migrate/database/redshift"
-)

cli/commands.go

@@ -1,81 +0,0 @@
-package main
-
-import (
-	"github.com/mattes/migrate"
-	_ "github.com/mattes/migrate/database/stub" // TODO remove again
-	_ "github.com/mattes/migrate/source/file"
-)
-
-func gotoCmd(m *migrate.Migrate, v uint) {
-	if err := m.Migrate(v); err != nil {
-		if err != migrate.ErrNoChange {
-			log.fatalErr(err)
-		} else {
-			log.Println(err)
-		}
-	}
-}
-
-func upCmd(m *migrate.Migrate, limit int) {
-	if limit >= 0 {
-		if err := m.Steps(limit); err != nil {
-			if err != migrate.ErrNoChange {
-				log.fatalErr(err)
-			} else {
-				log.Println(err)
-			}
-		}
-	} else {
-		if err := m.Up(); err != nil {
-			if err != migrate.ErrNoChange {
-				log.fatalErr(err)
-			} else {
-				log.Println(err)
-			}
-		}
-	}
-}
-
-func downCmd(m *migrate.Migrate, limit int) {
-	if limit >= 0 {
-		if err := m.Steps(-limit); err != nil {
-			if err != migrate.ErrNoChange {
-				log.fatalErr(err)
-			} else {
-				log.Println(err)
-			}
-		}
-	} else {
-		if err := m.Down(); err != nil {
-			if err != migrate.ErrNoChange {
-				log.fatalErr(err)
-			} else {
-				log.Println(err)
-			}
-		}
-	}
-}
-
-func dropCmd(m *migrate.Migrate) {
-	if err := m.Drop(); err != nil {
-		log.fatalErr(err)
-	}
-}
-
-func forceCmd(m *migrate.Migrate, v int) {
-	if err := m.Force(v); err != nil {
-		log.fatalErr(err)
-	}
-}
-
-func versionCmd(m *migrate.Migrate) {
-	v, dirty, err := m.Version()
-	if err != nil {
-		log.fatalErr(err)
-	}
-	if dirty {
-		log.Printf("%v (dirty)\n", v)
-	} else {
-		log.Println(v)
-	}
-}

cli/examples/Dockerfile

@@ -1,12 +0,0 @@
-FROM ubuntu:xenial
-
-RUN apt-get update && \
-    apt-get install -y curl apt-transport-https
-
-RUN curl -L https://packagecloud.io/mattes/migrate/gpgkey | apt-key add - && \
-    echo "deb https://packagecloud.io/mattes/migrate/ubuntu/ xenial main" > /etc/apt/sources.list.d/migrate.list && \
-    apt-get update && \
-    apt-get install -y migrate
-
-RUN migrate -version
-

cli/main.go

@@ -1,210 +1,8 @@
 package main
 
-import (
-	"flag"
-	"fmt"
-	"os"
-	"os/signal"
-	"strconv"
-	"syscall"
-	"time"
-
-	"github.com/mattes/migrate"
-)
-
-// set main log
-var log = &Log{}
+import "github.com/golang-migrate/migrate/v4/internal/cli"
 
+// Deprecated, please use cmd/migrate
 func main() {
-	helpPtr := flag.Bool("help", false, "")
-	versionPtr := flag.Bool("version", false, "")
-	verbosePtr := flag.Bool("verbose", false, "")
-	prefetchPtr := flag.Uint("prefetch", 10, "")
-	lockTimeoutPtr := flag.Uint("lock-timeout", 15, "")
-	pathPtr := flag.String("path", "", "")
-	databasePtr := flag.String("database", "", "")
-	sourcePtr := flag.String("source", "", "")
-
-	flag.Usage = func() {
-		fmt.Fprint(os.Stderr,
-			`Usage: migrate OPTIONS COMMAND [arg...]
-       migrate [ -version | -help ]
-
-Options:
-  -source          Location of the migrations (driver://url)
-  -path            Shorthand for -source=file://path 
-  -database        Run migrations against this database (driver://url)
-  -prefetch N      Number of migrations to load in advance before executing (default 10)
-  -lock-timeout N  Allow N seconds to acquire database lock (default 15)
-  -verbose         Print verbose logging
-  -version         Print version
-  -help            Print usage
-
-Commands:
-  goto V       Migrate to version V
-  up [N]       Apply all or N up migrations
-  down [N]     Apply all or N down migrations
-  drop         Drop everyting inside database
-  force V      Set version V but don't run migration (ignores dirty state)
-  version      Print current migration version
-`)
-	}
-
-	flag.Parse()
-
-	// initialize logger
-	log.verbose = *verbosePtr
-
-	// show cli version
-	if *versionPtr {
-		fmt.Fprintln(os.Stderr, Version)
-		os.Exit(0)
-	}
-
-	// show help
-	if *helpPtr {
-		flag.Usage()
-		os.Exit(0)
-	}
-
-	// translate -path into -source if given
-	if *sourcePtr == "" && *pathPtr != "" {
-		*sourcePtr = fmt.Sprintf("file://%v", *pathPtr)
-	}
-
-	// initialize migrate
-	// don't catch migraterErr here and let each command decide
-	// how it wants to handle the error
-	migrater, migraterErr := migrate.New(*sourcePtr, *databasePtr)
-	defer func() {
-		if migraterErr == nil {
-			migrater.Close()
-		}
-	}()
-	if migraterErr == nil {
-		migrater.Log = log
-		migrater.PrefetchMigrations = *prefetchPtr
-		migrater.LockTimeout = time.Duration(int64(*lockTimeoutPtr)) * time.Second
-
-		// handle Ctrl+c
-		signals := make(chan os.Signal, 1)
-		signal.Notify(signals, syscall.SIGINT)
-		go func() {
-			for range signals {
-				log.Println("Stopping after this running migration ...")
-				migrater.GracefulStop <- true
-				return
-			}
-		}()
-	}
-
-	startTime := time.Now()
-
-	switch flag.Arg(0) {
-	case "goto":
-		if migraterErr != nil {
-			log.fatalErr(migraterErr)
-		}
-
-		if flag.Arg(1) == "" {
-			log.fatal("error: please specify version argument V")
-		}
-
-		v, err := strconv.ParseUint(flag.Arg(1), 10, 64)
-		if err != nil {
-			log.fatal("error: can't read version argument V")
-		}
-
-		gotoCmd(migrater, uint(v))
-
-		if log.verbose {
-			log.Println("Finished after", time.Now().Sub(startTime))
-		}
-
-	case "up":
-		if migraterErr != nil {
-			log.fatalErr(migraterErr)
-		}
-
-		limit := -1
-		if flag.Arg(1) != "" {
-			n, err := strconv.ParseUint(flag.Arg(1), 10, 64)
-			if err != nil {
-				log.fatal("error: can't read limit argument N")
-			}
-			limit = int(n)
-		}
-
-		upCmd(migrater, limit)
-
-		if log.verbose {
-			log.Println("Finished after", time.Now().Sub(startTime))
-		}
-
-	case "down":
-		if migraterErr != nil {
-			log.fatalErr(migraterErr)
-		}
-
-		limit := -1
-		if flag.Arg(1) != "" {
-			n, err := strconv.ParseUint(flag.Arg(1), 10, 64)
-			if err != nil {
-				log.fatal("error: can't read limit argument N")
-			}
-			limit = int(n)
-		}
-
-		downCmd(migrater, limit)
-
-		if log.verbose {
-			log.Println("Finished after", time.Now().Sub(startTime))
-		}
-
-	case "drop":
-		if migraterErr != nil {
-			log.fatalErr(migraterErr)
-		}
-
-		dropCmd(migrater)
-
-		if log.verbose {
-			log.Println("Finished after", time.Now().Sub(startTime))
-		}
-
-	case "force":
-		if migraterErr != nil {
-			log.fatalErr(migraterErr)
-		}
-
-		if flag.Arg(1) == "" {
-			log.fatal("error: please specify version argument V")
-		}
-
-		v, err := strconv.ParseInt(flag.Arg(1), 10, 64)
-		if err != nil {
-			log.fatal("error: can't read version argument V")
-		}
-
-		if v < -1 {
-			log.fatal("error: argument V must be >= -1")
-		}
-
-		forceCmd(migrater, int(v))
-
-		if log.verbose {
-			log.Println("Finished after", time.Now().Sub(startTime))
-		}
-
-	case "version":
-		if migraterErr != nil {
-			log.fatalErr(migraterErr)
-		}
-
-		versionCmd(migrater)
-
-	default:
-		flag.Usage()
-		os.Exit(0)
-	}
+	cli.Main(Version)
 }

cmd/migrate/README.md

@@ -0,0 +1,123 @@
+# migrate CLI
+
+## Installation
+
+### Download pre-built binary (Windows, MacOS, or Linux)
+
+[Release Downloads](https://github.com/golang-migrate/migrate/releases)
+
+```bash
+$ curl -L https://github.com/golang-migrate/migrate/releases/download/$version/migrate.$platform-amd64.tar.gz | tar xvz
+```
+
+### MacOS
+
+```bash
+$ brew install golang-migrate
+```
+
+### Linux (*.deb package)
+
+```bash
+$ curl -L https://packagecloud.io/golang-migrate/migrate/gpgkey | apt-key add -
+$ echo "deb https://packagecloud.io/golang-migrate/migrate/ubuntu/ $(lsb_release -sc) main" > /etc/apt/sources.list.d/migrate.list
+$ apt-get update
+$ apt-get install -y migrate
+```
+
+### With Go toolchain
+
+#### Versioned
+
+```bash
+$ go get -u -d github.com/golang-migrate/migrate/cmd/migrate
+$ cd $GOPATH/src/github.com/golang-migrate/migrate/cmd/migrate
+$ git checkout $TAG  # e.g. v4.1.0
+$ go build -tags 'postgres' -ldflags="-X main.Version=$(git describe --tags)" -o $GOPATH/bin/migrate github.com/golang-migrate/migrate/cmd/migrate
+```
+
+#### Unversioned
+
+```bash
+$ go get -tags 'postgres' -u github.com/golang-migrate/migrate/cmd/migrate
+```
+
+#### Notes
+
+1. Requires a version of Go that [supports modules](https://golang.org/cmd/go/#hdr-Preliminary_module_support). e.g. Go 1.11+
+1. These examples build the cli which will only work with postgres.  In order
+to build the cli for use with other databases, replace the `postgres` build tag
+with the appropriate database tag(s) for the databases desired.  The tags
+correspond to the names of the sub-packages underneath the
+[`database`](../database) package.
+1. Similarly to the database build tags, if you need to support other sources, use the appropriate build tag(s).
+1. Support for build constraints will be removed in the future: https://github.com/golang-migrate/migrate/issues/60
+
+## Usage
+
+```bash
+$ migrate -help
+Usage: migrate OPTIONS COMMAND [arg...]
+       migrate [ -version | -help ]
+
+Options:
+  -source          Location of the migrations (driver://url)
+  -path            Shorthand for -source=file://path
+  -database        Run migrations against this database (driver://url)
+  -prefetch N      Number of migrations to load in advance before executing (default 10)
+  -lock-timeout N  Allow N seconds to acquire database lock (default 15)
+  -verbose         Print verbose logging
+  -version         Print version
+  -help            Print usage
+
+Commands:
+  create [-ext E] [-dir D] [-seq] [-digits N] [-format] NAME
+               Create a set of timestamped up/down migrations titled NAME, in directory D with extension E.
+               Use -seq option to generate sequential up/down migrations with N digits.
+               Use -format option to specify a Go time format string.
+  goto V       Migrate to version V
+  up [N]       Apply all or N up migrations
+  down [N]     Apply all or N down migrations
+  drop         Drop everything inside database
+  force V      Set version V but don't run migration (ignores dirty state)
+  version      Print current migration version
+```
+
+So let's say you want to run the first two migrations
+
+```bash
+$ migrate -source file://path/to/migrations -database postgres://localhost:5432/database up 2
+```
+
+If your migrations are hosted on github
+
+```bash
+$ migrate -source github://mattes:personal-access-token@mattes/migrate_test \
+    -database postgres://localhost:5432/database down 2
+```
+
+The CLI will gracefully stop at a safe point when SIGINT (ctrl+c) is received.
+Send SIGKILL for immediate halt.
+
+## Reading CLI arguments from somewhere else
+
+### ENV variables
+
+```bash
+$ migrate -database "$MY_MIGRATE_DATABASE"
+```
+
+### JSON files
+
+Check out https://stedolan.github.io/jq/
+
+```bash
+$ migrate -database "$(cat config.json | jq '.database')"
+```
+
+### YAML files
+
+```bash
+$ migrate -database "$(cat config/database.yml | ruby -ryaml -e "print YAML.load(STDIN.read)['database']")"
+$ migrate -database "$(cat config/database.yml | python -c 'import yaml,sys;print yaml.safe_load(sys.stdin)["database"]')"
+```

cmd/migrate/examples/Dockerfile

@@ -0,0 +1,15 @@
+FROM ubuntu:bionic
+
+RUN apt-get update && \
+    apt-get install -y \
+    apt-transport-https \
+    ca-certificates \
+    curl \
+    gnupg-agent
+
+RUN curl -sSL https://packagecloud.io/golang-migrate/migrate/gpgkey | apt-key add -
+RUN echo "deb https://packagecloud.io/golang-migrate/migrate/ubuntu/ bionic main" > /etc/apt/sources.list.d/migrate.list
+RUN apt-get update && \
+    apt-get install -y migrate
+
+RUN migrate -version

cmd/migrate/main.go

@@ -0,0 +1,7 @@
+package main
+
+import "github.com/golang-migrate/migrate/v4/internal/cli"
+
+func main() {
+	cli.Main(Version)
+}

cmd/migrate/version.go

@@ -0,0 +1,4 @@
+package main
+
+// Version is set in Makefile with build flags
+var Version = "dev"

database/cassandra/README.md

@@ -0,0 +1,38 @@
+# Cassandra
+
+* Drop command will not work on Cassandra 2.X because it rely on
+system_schema table which comes with 3.X
+* Other commands should work properly but are **not tested**
+* The Cassandra driver (gocql) does not natively support executing multipe statements in a single query. To allow for multiple statements in a single migration, you can use the `x-multi-statement` param. There are two important caveats:
+  * This mode splits the migration text into separately-executed statements by a semi-colon `;`. Thus `x-multi-statement` cannot be used when a statement in the migration contains a string with a semi-colon.
+  * The queries are not executed in any sort of transaction/batch, meaning you are responsible for fixing partial migrations.
+
+
+## Usage
+`cassandra://host:port/keyspace?param1=value&param2=value2`
+
+
+| URL Query  | Default value | Description |
+|------------|-------------|-----------|
+| `x-migrations-table` | schema_migrations | Name of the migrations table |
+| `x-multi-statement` | false | Enable multiple statements to be ran in a single migration (See note above) |
+| `port` | 9042 | The port to bind to  |
+| `consistency` | ALL | Migration consistency
+| `protocol` |  | Cassandra protocol version (3 or 4)
+| `timeout` | 1 minute | Migration timeout
+| `username` | nil | Username to use when authenticating. |
+| `password` | nil | Password to use when authenticating. |
+| `sslcert` | | Cert file location. The file must contain PEM encoded data. |
+| `sslkey` | | Key file location. The file must contain PEM encoded data. |
+| `sslrootcert` | | The location of the root certificate file. The file must contain PEM encoded data. |
+| `sslmode` | | Whether or not to use SSL (disable\|require\|verify-ca\|verify-full) |
+
+`timeout` is parsed using [time.ParseDuration(s string)](https://golang.org/pkg/time/#ParseDuration)
+
+
+## Upgrading from v1
+
+1. Write down the current migration version from schema_migrations
+2. `DROP TABLE schema_migrations`
+4. Download and install the latest migrate version.
+5. Force the current migration version with `migrate force <current_version>`.

database/cassandra/cassandra.go

@@ -0,0 +1,291 @@
+package cassandra
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	nurl "net/url"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/gocql/gocql"
+	"github.com/golang-migrate/migrate/v4/database"
+	"github.com/hashicorp/go-multierror"
+)
+
+func init() {
+	db := new(Cassandra)
+	database.Register("cassandra", db)
+}
+
+var DefaultMigrationsTable = "schema_migrations"
+
+var (
+	ErrNilConfig     = errors.New("no config")
+	ErrNoKeyspace    = errors.New("no keyspace provided")
+	ErrDatabaseDirty = errors.New("database is dirty")
+	ErrClosedSession = errors.New("session is closed")
+)
+
+type Config struct {
+	MigrationsTable       string
+	KeyspaceName          string
+	MultiStatementEnabled bool
+}
+
+type Cassandra struct {
+	session  *gocql.Session
+	isLocked bool
+
+	// Open and WithInstance need to guarantee that config is never nil
+	config *Config
+}
+
+func WithInstance(session *gocql.Session, config *Config) (database.Driver, error) {
+	if config == nil {
+		return nil, ErrNilConfig
+	} else if len(config.KeyspaceName) == 0 {
+		return nil, ErrNoKeyspace
+	}
+
+	if session.Closed() {
+		return nil, ErrClosedSession
+	}
+
+	if len(config.MigrationsTable) == 0 {
+		config.MigrationsTable = DefaultMigrationsTable
+	}
+
+	c := &Cassandra{
+		session: session,
+		config:  config,
+	}
+
+	if err := c.ensureVersionTable(); err != nil {
+		return nil, err
+	}
+
+	return c, nil
+}
+
+func (c *Cassandra) Open(url string) (database.Driver, error) {
+	u, err := nurl.Parse(url)
+	if err != nil {
+		return nil, err
+	}
+
+	// Check for missing mandatory attributes
+	if len(u.Path) == 0 {
+		return nil, ErrNoKeyspace
+	}
+
+	cluster := gocql.NewCluster(u.Host)
+	cluster.Keyspace = strings.TrimPrefix(u.Path, "/")
+	cluster.Consistency = gocql.All
+	cluster.Timeout = 1 * time.Minute
+
+	if len(u.Query().Get("username")) > 0 && len(u.Query().Get("password")) > 0 {
+		authenticator := gocql.PasswordAuthenticator{
+			Username: u.Query().Get("username"),
+			Password: u.Query().Get("password"),
+		}
+		cluster.Authenticator = authenticator
+	}
+
+	// Retrieve query string configuration
+	if len(u.Query().Get("consistency")) > 0 {
+		var consistency gocql.Consistency
+		consistency, err = parseConsistency(u.Query().Get("consistency"))
+		if err != nil {
+			return nil, err
+		}
+
+		cluster.Consistency = consistency
+	}
+	if len(u.Query().Get("protocol")) > 0 {
+		var protoversion int
+		protoversion, err = strconv.Atoi(u.Query().Get("protocol"))
+		if err != nil {
+			return nil, err
+		}
+		cluster.ProtoVersion = protoversion
+	}
+	if len(u.Query().Get("timeout")) > 0 {
+		var timeout time.Duration
+		timeout, err = time.ParseDuration(u.Query().Get("timeout"))
+		if err != nil {
+			return nil, err
+		}
+		cluster.Timeout = timeout
+	}
+
+	if len(u.Query().Get("sslmode")) > 0 && len(u.Query().Get("sslrootcert")) > 0 && len(u.Query().Get("sslcert")) > 0 && len(u.Query().Get("sslkey")) > 0 {
+		if u.Query().Get("sslmode") != "disable" {
+			cluster.SslOpts = &gocql.SslOptions{
+				CaPath:   u.Query().Get("sslrootcert"),
+				CertPath: u.Query().Get("sslcert"),
+				KeyPath:  u.Query().Get("sslkey"),
+			}
+			if u.Query().Get("sslmode") == "verify-full" {
+				cluster.SslOpts.EnableHostVerification = true
+			}
+		}
+	}
+
+	session, err := cluster.CreateSession()
+	if err != nil {
+		return nil, err
+	}
+
+	return WithInstance(session, &Config{
+		KeyspaceName:          strings.TrimPrefix(u.Path, "/"),
+		MigrationsTable:       u.Query().Get("x-migrations-table"),
+		MultiStatementEnabled: u.Query().Get("x-multi-statement") == "true",
+	})
+}
+
+func (c *Cassandra) Close() error {
+	c.session.Close()
+	return nil
+}
+
+func (c *Cassandra) Lock() error {
+	if c.isLocked {
+		return database.ErrLocked
+	}
+	c.isLocked = true
+	return nil
+}
+
+func (c *Cassandra) Unlock() error {
+	c.isLocked = false
+	return nil
+}
+
+func (c *Cassandra) Run(migration io.Reader) error {
+	migr, err := ioutil.ReadAll(migration)
+	if err != nil {
+		return err
+	}
+	// run migration
+	query := string(migr[:])
+
+	if c.config.MultiStatementEnabled {
+		// split query by semi-colon
+		queries := strings.Split(query, ";")
+
+		for _, q := range queries {
+			tq := strings.TrimSpace(q)
+			if tq == "" {
+				continue
+			}
+			if err := c.session.Query(tq).Exec(); err != nil {
+				// TODO: cast to Cassandra error and get line number
+				return database.Error{OrigErr: err, Err: "migration failed", Query: migr}
+			}
+		}
+		return nil
+	}
+
+	if err := c.session.Query(query).Exec(); err != nil {
+		// TODO: cast to Cassandra error and get line number
+		return database.Error{OrigErr: err, Err: "migration failed", Query: migr}
+	}
+	return nil
+}
+
+func (c *Cassandra) SetVersion(version int, dirty bool) error {
+	query := `TRUNCATE "` + c.config.MigrationsTable + `"`
+	if err := c.session.Query(query).Exec(); err != nil {
+		return &database.Error{OrigErr: err, Query: []byte(query)}
+	}
+	if version >= 0 {
+		query = `INSERT INTO "` + c.config.MigrationsTable + `" (version, dirty) VALUES (?, ?)`
+		if err := c.session.Query(query, version, dirty).Exec(); err != nil {
+			return &database.Error{OrigErr: err, Query: []byte(query)}
+		}
+	}
+
+	return nil
+}
+
+// Return current keyspace version
+func (c *Cassandra) Version() (version int, dirty bool, err error) {
+	query := `SELECT version, dirty FROM "` + c.config.MigrationsTable + `" LIMIT 1`
+	err = c.session.Query(query).Scan(&version, &dirty)
+	switch {
+	case err == gocql.ErrNotFound:
+		return database.NilVersion, false, nil
+
+	case err != nil:
+		if _, ok := err.(*gocql.Error); ok {
+			return database.NilVersion, false, nil
+		}
+		return 0, false, &database.Error{OrigErr: err, Query: []byte(query)}
+
+	default:
+		return version, dirty, nil
+	}
+}
+
+func (c *Cassandra) Drop() error {
+	// select all tables in current schema
+	query := fmt.Sprintf(`SELECT table_name from system_schema.tables WHERE keyspace_name='%s'`, c.config.KeyspaceName)
+	iter := c.session.Query(query).Iter()
+	var tableName string
+	for iter.Scan(&tableName) {
+		err := c.session.Query(fmt.Sprintf(`DROP TABLE %s`, tableName)).Exec()
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// ensureVersionTable checks if versions table exists and, if not, creates it.
+// Note that this function locks the database, which deviates from the usual
+// convention of "caller locks" in the Cassandra type.
+func (c *Cassandra) ensureVersionTable() (err error) {
+	if err = c.Lock(); err != nil {
+		return err
+	}
+
+	defer func() {
+		if e := c.Unlock(); e != nil {
+			if err == nil {
+				err = e
+			} else {
+				err = multierror.Append(err, e)
+			}
+		}
+	}()
+
+	err = c.session.Query(fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (version bigint, dirty boolean, PRIMARY KEY(version))", c.config.MigrationsTable)).Exec()
+	if err != nil {
+		return err
+	}
+	if _, _, err = c.Version(); err != nil {
+		return err
+	}
+	return nil
+}
+
+// ParseConsistency wraps gocql.ParseConsistency
+// to return an error instead of a panicking.
+func parseConsistency(consistencyStr string) (consistency gocql.Consistency, err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			var ok bool
+			err, ok = r.(error)
+			if !ok {
+				err = fmt.Errorf("Failed to parse consistency \"%s\": %v", consistencyStr, r)
+			}
+		}
+	}()
+	consistency = gocql.ParseConsistency(consistencyStr)
+
+	return consistency, nil
+}

database/cassandra/cassandra_test.go

@@ -0,0 +1,106 @@
+package cassandra
+
+import (
+	"context"
+	"fmt"
+	"github.com/golang-migrate/migrate/v4"
+	"strconv"
+	"testing"
+)
+
+import (
+	"github.com/dhui/dktest"
+	"github.com/gocql/gocql"
+)
+
+import (
+	dt "github.com/golang-migrate/migrate/v4/database/testing"
+	"github.com/golang-migrate/migrate/v4/dktesting"
+	_ "github.com/golang-migrate/migrate/v4/source/file"
+)
+
+var (
+	opts = dktest.Options{PortRequired: true, ReadyFunc: isReady}
+	// Supported versions: http://cassandra.apache.org/download/
+	// Although Cassandra 2.x is supported by the Apache Foundation,
+	// the migrate db driver only supports Cassandra 3.x since it uses
+	// the system_schema keyspace.
+	specs = []dktesting.ContainerSpec{
+		{ImageName: "cassandra:3.0", Options: opts},
+		{ImageName: "cassandra:3.11", Options: opts},
+	}
+)
+
+func isReady(ctx context.Context, c dktest.ContainerInfo) bool {
+	// Cassandra exposes 5 ports (7000, 7001, 7199, 9042 & 9160)
+	// We only need the port bound to 9042
+	ip, portStr, err := c.Port(9042)
+	if err != nil {
+		return false
+	}
+	port, err := strconv.Atoi(portStr)
+	if err != nil {
+		return false
+	}
+
+	cluster := gocql.NewCluster(ip)
+	cluster.Port = port
+	cluster.Consistency = gocql.All
+	p, err := cluster.CreateSession()
+	if err != nil {
+		return false
+	}
+	defer p.Close()
+	// Create keyspace for tests
+	if err = p.Query("CREATE KEYSPACE testks WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor':1}").Exec(); err != nil {
+		return false
+	}
+	return true
+}
+
+func Test(t *testing.T) {
+	dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
+		ip, port, err := c.Port(9042)
+		if err != nil {
+			t.Fatal("Unable to get mapped port:", err)
+		}
+		addr := fmt.Sprintf("cassandra://%v:%v/testks", ip, port)
+		p := &Cassandra{}
+		d, err := p.Open(addr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer func() {
+			if err := d.Close(); err != nil {
+				t.Error(err)
+			}
+		}()
+		dt.Test(t, d, []byte("SELECT table_name from system_schema.tables"))
+	})
+}
+
+func TestMigrate(t *testing.T) {
+	dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
+		ip, port, err := c.Port(9042)
+		if err != nil {
+			t.Fatal("Unable to get mapped port:", err)
+		}
+		addr := fmt.Sprintf("cassandra://%v:%v/testks", ip, port)
+		p := &Cassandra{}
+		d, err := p.Open(addr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer func() {
+			if err := d.Close(); err != nil {
+				t.Error(err)
+			}
+		}()
+
+		m, err := migrate.NewWithDatabaseInstance("file://./examples/migrations", "testks", d)
+		if err != nil {
+			t.Fatal(err)
+		}
+		dt.TestMigrate(t, m, []byte("SELECT table_name from system_schema.tables"))
+	})
+}

database/cassandra/examples/migrations/1_simple_select.down.sql

@@ -0,0 +1 @@
+SELECT table_name from system_schema.tables

database/cassandra/examples/migrations/1_simple_select.up.sql

@@ -0,0 +1 @@
+SELECT table_name from system_schema.tables

database/clickhouse/README.md

@@ -0,0 +1,19 @@
+# ClickHouse
+
+`clickhouse://host:port?username=user&password=qwerty&database=clicks&x-multi-statement=true`
+
+| URL Query  | Description |
+|------------|-------------|
+| `x-migrations-table`| Name of the migrations table |
+| `database` | The name of the database to connect to |
+| `username` | The user to sign in as |
+| `password` | The user's password | 
+| `host` | The host to connect to. |
+| `port` | The port to bind to. |
+| `x-multi-statement` | false | Enable multiple statements to be ran in a single migration (See note below) |
+
+## Notes
+
+* The Clickhouse driver does not natively support executing multipe statements in a single query. To allow for multiple statements in a single migration, you can use the `x-multi-statement` param. There are two important caveats:
+  * This mode splits the migration text into separately-executed statements by a semi-colon `;`. Thus `x-multi-statement` cannot be used when a statement in the migration contains a string with a semi-colon.
+  * The queries are not executed in any sort of transaction/batch, meaning you are responsible for fixing partial migrations.

database/clickhouse/clickhouse.go

@@ -0,0 +1,236 @@
+package clickhouse
+
+import (
+	"database/sql"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/golang-migrate/migrate/v4"
+	"github.com/golang-migrate/migrate/v4/database"
+	"github.com/hashicorp/go-multierror"
+)
+
+var DefaultMigrationsTable = "schema_migrations"
+
+var ErrNilConfig = fmt.Errorf("no config")
+
+type Config struct {
+	DatabaseName          string
+	MigrationsTable       string
+	MultiStatementEnabled bool
+}
+
+func init() {
+	database.Register("clickhouse", &ClickHouse{})
+}
+
+func WithInstance(conn *sql.DB, config *Config) (database.Driver, error) {
+	if config == nil {
+		return nil, ErrNilConfig
+	}
+
+	if err := conn.Ping(); err != nil {
+		return nil, err
+	}
+
+	ch := &ClickHouse{
+		conn:   conn,
+		config: config,
+	}
+
+	if err := ch.init(); err != nil {
+		return nil, err
+	}
+
+	return ch, nil
+}
+
+type ClickHouse struct {
+	conn   *sql.DB
+	config *Config
+}
+
+func (ch *ClickHouse) Open(dsn string) (database.Driver, error) {
+	purl, err := url.Parse(dsn)
+	if err != nil {
+		return nil, err
+	}
+	q := migrate.FilterCustomQuery(purl)
+	q.Scheme = "tcp"
+	conn, err := sql.Open("clickhouse", q.String())
+	if err != nil {
+		return nil, err
+	}
+
+	ch = &ClickHouse{
+		conn: conn,
+		config: &Config{
+			MigrationsTable:       purl.Query().Get("x-migrations-table"),
+			DatabaseName:          purl.Query().Get("database"),
+			MultiStatementEnabled: purl.Query().Get("x-multi-statement") == "true",
+		},
+	}
+
+	if err := ch.init(); err != nil {
+		return nil, err
+	}
+
+	return ch, nil
+}
+
+func (ch *ClickHouse) init() error {
+	if len(ch.config.DatabaseName) == 0 {
+		if err := ch.conn.QueryRow("SELECT currentDatabase()").Scan(&ch.config.DatabaseName); err != nil {
+			return err
+		}
+	}
+
+	if len(ch.config.MigrationsTable) == 0 {
+		ch.config.MigrationsTable = DefaultMigrationsTable
+	}
+
+	return ch.ensureVersionTable()
+}
+
+func (ch *ClickHouse) Run(r io.Reader) error {
+	migration, err := ioutil.ReadAll(r)
+	if err != nil {
+		return err
+	}
+
+	if ch.config.MultiStatementEnabled {
+		// split query by semi-colon
+		queries := strings.Split(string(migration), ";")
+		for _, q := range queries {
+			tq := strings.TrimSpace(q)
+			if tq == "" {
+				continue
+			}
+			if _, err := ch.conn.Exec(q); err != nil {
+				return database.Error{OrigErr: err, Err: "migration failed", Query: []byte(q)}
+			}
+		}
+		return nil
+	}
+
+	if _, err := ch.conn.Exec(string(migration)); err != nil {
+		return database.Error{OrigErr: err, Err: "migration failed", Query: migration}
+	}
+
+	return nil
+}
+func (ch *ClickHouse) Version() (int, bool, error) {
+	var (
+		version int
+		dirty   uint8
+		query   = "SELECT version, dirty FROM `" + ch.config.MigrationsTable + "` ORDER BY sequence DESC LIMIT 1"
+	)
+	if err := ch.conn.QueryRow(query).Scan(&version, &dirty); err != nil {
+		if err == sql.ErrNoRows {
+			return database.NilVersion, false, nil
+		}
+		return 0, false, &database.Error{OrigErr: err, Query: []byte(query)}
+	}
+	return version, dirty == 1, nil
+}
+
+func (ch *ClickHouse) SetVersion(version int, dirty bool) error {
+	var (
+		bool = func(v bool) uint8 {
+			if v {
+				return 1
+			}
+			return 0
+		}
+		tx, err = ch.conn.Begin()
+	)
+	if err != nil {
+		return err
+	}
+
+	query := "INSERT INTO " + ch.config.MigrationsTable + " (version, dirty, sequence) VALUES (?, ?, ?)"
+	if _, err := tx.Exec(query, version, bool(dirty), time.Now().UnixNano()); err != nil {
+		return &database.Error{OrigErr: err, Query: []byte(query)}
+	}
+
+	return tx.Commit()
+}
+
+// ensureVersionTable checks if versions table exists and, if not, creates it.
+// Note that this function locks the database, which deviates from the usual
+// convention of "caller locks" in the ClickHouse type.
+func (ch *ClickHouse) ensureVersionTable() (err error) {
+	if err = ch.Lock(); err != nil {
+		return err
+	}
+
+	defer func() {
+		if e := ch.Unlock(); e != nil {
+			if err == nil {
+				err = e
+			} else {
+				err = multierror.Append(err, e)
+			}
+		}
+	}()
+
+	var (
+		table string
+		query = "SHOW TABLES FROM " + ch.config.DatabaseName + " LIKE '" + ch.config.MigrationsTable + "'"
+	)
+	// check if migration table exists
+	if err := ch.conn.QueryRow(query).Scan(&table); err != nil {
+		if err != sql.ErrNoRows {
+			return &database.Error{OrigErr: err, Query: []byte(query)}
+		}
+	} else {
+		return nil
+	}
+	// if not, create the empty migration table
+	query = `
+		CREATE TABLE ` + ch.config.MigrationsTable + ` (
+			version    Int64, 
+			dirty      UInt8,
+			sequence   UInt64
+		) Engine=TinyLog
+	`
+	if _, err := ch.conn.Exec(query); err != nil {
+		return &database.Error{OrigErr: err, Query: []byte(query)}
+	}
+	return nil
+}
+
+func (ch *ClickHouse) Drop() (err error) {
+	query := "SHOW TABLES FROM " + ch.config.DatabaseName
+	tables, err := ch.conn.Query(query)
+
+	if err != nil {
+		return &database.Error{OrigErr: err, Query: []byte(query)}
+	}
+	defer func() {
+		if errClose := tables.Close(); errClose != nil {
+			err = multierror.Append(err, errClose)
+		}
+	}()
+	for tables.Next() {
+		var table string
+		if err := tables.Scan(&table); err != nil {
+			return err
+		}
+
+		query = "DROP TABLE IF EXISTS " + ch.config.DatabaseName + "." + table
+
+		if _, err := ch.conn.Exec(query); err != nil {
+			return &database.Error{OrigErr: err, Query: []byte(query)}
+		}
+	}
+	return nil
+}
+
+func (ch *ClickHouse) Lock() error   { return nil }
+func (ch *ClickHouse) Unlock() error { return nil }
+func (ch *ClickHouse) Close() error  { return ch.conn.Close() }

database/clickhouse/examples/migrations/001_init.down.sql

@@ -0,0 +1 @@
+DROP TABLE IF EXISTS test_1;

database/clickhouse/examples/migrations/001_init.up.sql

@@ -0,0 +1,3 @@
+CREATE TABLE test_1 (
+    Date Date 
+) Engine=Memory;

database/clickhouse/examples/migrations/002_create_table.down.sql

@@ -0,0 +1 @@
+DROP TABLE IF EXISTS test_2;

database/clickhouse/examples/migrations/002_create_table.up.sql

@@ -0,0 +1,3 @@
+CREATE TABLE test_2 (
+    Date Date 
+) Engine=Memory;

database/cockroachdb/README.md

@@ -0,0 +1,19 @@
+# cockroachdb
+
+`cockroachdb://user:password@host:port/dbname?query` (`cockroach://`, and `crdb-postgres://` work, too)
+
+| URL Query  | WithInstance Config | Description |
+|------------|---------------------|-------------|
+| `x-migrations-table` | `MigrationsTable` | Name of the migrations table |
+| `x-lock-table` | `LockTable` | Name of the table which maintains the migration lock |
+| `x-force-lock` | `ForceLock` | Force lock acquisition to fix faulty migrations which may not have released the schema lock (Boolean, default is `false`) |
+| `dbname` | `DatabaseName` | The name of the database to connect to |
+| `user` | | The user to sign in as |
+| `password` | | The user's password |
+| `host` | | The host to connect to. Values that start with / are for unix domain sockets. (default is localhost) |
+| `port` | | The port to bind to. (default is 5432) |
+| `connect_timeout` | | Maximum wait for connection, in seconds. Zero or not specified means wait indefinitely. |
+| `sslcert` | | Cert file location. The file must contain PEM encoded data. |
+| `sslkey` | | Key file location. The file must contain PEM encoded data. |
+| `sslrootcert` | | The location of the root certificate file. The file must contain PEM encoded data. |
+| `sslmode` | | Whether or not to use SSL (disable\|require\|verify-ca\|verify-full) |

database/cockroachdb/cockroachdb.go

@@ -0,0 +1,368 @@
+package cockroachdb
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"io"
+	"io/ioutil"
+	nurl "net/url"
+	"regexp"
+	"strconv"
+)
+
+import (
+	"github.com/cockroachdb/cockroach-go/crdb"
+	"github.com/hashicorp/go-multierror"
+	"github.com/lib/pq"
+)
+
+import (
+	"github.com/golang-migrate/migrate/v4"
+	"github.com/golang-migrate/migrate/v4/database"
+)
+
+func init() {
+	db := CockroachDb{}
+	database.Register("cockroach", &db)
+	database.Register("cockroachdb", &db)
+	database.Register("crdb-postgres", &db)
+}
+
+var DefaultMigrationsTable = "schema_migrations"
+var DefaultLockTable = "schema_lock"
+
+var (
+	ErrNilConfig      = fmt.Errorf("no config")
+	ErrNoDatabaseName = fmt.Errorf("no database name")
+)
+
+type Config struct {
+	MigrationsTable string
+	LockTable       string
+	ForceLock       bool
+	DatabaseName    string
+}
+
+type CockroachDb struct {
+	db       *sql.DB
+	isLocked bool
+
+	// Open and WithInstance need to guarantee that config is never nil
+	config *Config
+}
+
+func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) {
+	if config == nil {
+		return nil, ErrNilConfig
+	}
+
+	if err := instance.Ping(); err != nil {
+		return nil, err
+	}
+
+	if config.DatabaseName == "" {
+		query := `SELECT current_database()`
+		var databaseName string
+		if err := instance.QueryRow(query).Scan(&databaseName); err != nil {
+			return nil, &database.Error{OrigErr: err, Query: []byte(query)}
+		}
+
+		if len(databaseName) == 0 {
+			return nil, ErrNoDatabaseName
+		}
+
+		config.DatabaseName = databaseName
+	}
+
+	if len(config.MigrationsTable) == 0 {
+		config.MigrationsTable = DefaultMigrationsTable
+	}
+
+	if len(config.LockTable) == 0 {
+		config.LockTable = DefaultLockTable
+	}
+
+	px := &CockroachDb{
+		db:     instance,
+		config: config,
+	}
+
+	// ensureVersionTable is a locking operation, so we need to ensureLockTable before we ensureVersionTable.
+	if err := px.ensureLockTable(); err != nil {
+		return nil, err
+	}
+
+	if err := px.ensureVersionTable(); err != nil {
+		return nil, err
+	}
+
+	return px, nil
+}
+
+func (c *CockroachDb) Open(url string) (database.Driver, error) {
+	purl, err := nurl.Parse(url)
+	if err != nil {
+		return nil, err
+	}
+
+	// As Cockroach uses the postgres protocol, and 'postgres' is already a registered database, we need to replace the
+	// connect prefix, with the actual protocol, so that the library can differentiate between the implementations
+	re := regexp.MustCompile("^(cockroach(db)?|crdb-postgres)")
+	connectString := re.ReplaceAllString(migrate.FilterCustomQuery(purl).String(), "postgres")
+
+	db, err := sql.Open("postgres", connectString)
+	if err != nil {
+		return nil, err
+	}
+
+	migrationsTable := purl.Query().Get("x-migrations-table")
+	if len(migrationsTable) == 0 {
+		migrationsTable = DefaultMigrationsTable
+	}
+
+	lockTable := purl.Query().Get("x-lock-table")
+	if len(lockTable) == 0 {
+		lockTable = DefaultLockTable
+	}
+
+	forceLockQuery := purl.Query().Get("x-force-lock")
+	forceLock, err := strconv.ParseBool(forceLockQuery)
+	if err != nil {
+		forceLock = false
+	}
+
+	px, err := WithInstance(db, &Config{
+		DatabaseName:    purl.Path,
+		MigrationsTable: migrationsTable,
+		LockTable:       lockTable,
+		ForceLock:       forceLock,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return px, nil
+}
+
+func (c *CockroachDb) Close() error {
+	return c.db.Close()
+}
+
+// Locking is done manually with a separate lock table.  Implementing advisory locks in CRDB is being discussed
+// See: https://github.com/cockroachdb/cockroach/issues/13546
+func (c *CockroachDb) Lock() error {
+	err := crdb.ExecuteTx(context.Background(), c.db, nil, func(tx *sql.Tx) (err error) {
+		aid, err := database.GenerateAdvisoryLockId(c.config.DatabaseName)
+		if err != nil {
+			return err
+		}
+
+		query := "SELECT * FROM " + c.config.LockTable + " WHERE lock_id = $1"
+		rows, err := tx.Query(query, aid)
+		if err != nil {
+			return database.Error{OrigErr: err, Err: "failed to fetch migration lock", Query: []byte(query)}
+		}
+		defer func() {
+			if errClose := rows.Close(); errClose != nil {
+				err = multierror.Append(err, errClose)
+			}
+		}()
+
+		// If row exists at all, lock is present
+		locked := rows.Next()
+		if locked && !c.config.ForceLock {
+			return database.ErrLocked
+		}
+
+		query = "INSERT INTO " + c.config.LockTable + " (lock_id) VALUES ($1)"
+		if _, err := tx.Exec(query, aid); err != nil {
+			return database.Error{OrigErr: err, Err: "failed to set migration lock", Query: []byte(query)}
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return err
+	} else {
+		c.isLocked = true
+		return nil
+	}
+}
+
+// Locking is done manually with a separate lock table.  Implementing advisory locks in CRDB is being discussed
+// See: https://github.com/cockroachdb/cockroach/issues/13546
+func (c *CockroachDb) Unlock() error {
+	aid, err := database.GenerateAdvisoryLockId(c.config.DatabaseName)
+	if err != nil {
+		return err
+	}
+
+	// In the event of an implementation (non-migration) error, it is possible for the lock to not be released.  Until
+	// a better locking mechanism is added, a manual purging of the lock table may be required in such circumstances
+	query := "DELETE FROM " + c.config.LockTable + " WHERE lock_id = $1"
+	if _, err := c.db.Exec(query, aid); err != nil {
+		if e, ok := err.(*pq.Error); ok {
+			// 42P01 is "UndefinedTableError" in CockroachDB
+			// https://github.com/cockroachdb/cockroach/blob/master/pkg/sql/pgwire/pgerror/codes.go
+			if e.Code == "42P01" {
+				// On drops, the lock table is fully removed;  This is fine, and is a valid "unlocked" state for the schema
+				c.isLocked = false
+				return nil
+			}
+		}
+		return database.Error{OrigErr: err, Err: "failed to release migration lock", Query: []byte(query)}
+	}
+
+	c.isLocked = false
+	return nil
+}
+
+func (c *CockroachDb) Run(migration io.Reader) error {
+	migr, err := ioutil.ReadAll(migration)
+	if err != nil {
+		return err
+	}
+
+	// run migration
+	query := string(migr[:])
+	if _, err := c.db.Exec(query); err != nil {
+		return database.Error{OrigErr: err, Err: "migration failed", Query: migr}
+	}
+
+	return nil
+}
+
+func (c *CockroachDb) SetVersion(version int, dirty bool) error {
+	return crdb.ExecuteTx(context.Background(), c.db, nil, func(tx *sql.Tx) error {
+		if _, err := tx.Exec(`DELETE FROM "` + c.config.MigrationsTable + `"`); err != nil {
+			return err
+		}
+
+		if version >= 0 {
+			if _, err := tx.Exec(`INSERT INTO "`+c.config.MigrationsTable+`" (version, dirty) VALUES ($1, $2)`, version, dirty); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
+}
+
+func (c *CockroachDb) Version() (version int, dirty bool, err error) {
+	query := `SELECT version, dirty FROM "` + c.config.MigrationsTable + `" LIMIT 1`
+	err = c.db.QueryRow(query).Scan(&version, &dirty)
+
+	switch {
+	case err == sql.ErrNoRows:
+		return database.NilVersion, false, nil
+
+	case err != nil:
+		if e, ok := err.(*pq.Error); ok {
+			// 42P01 is "UndefinedTableError" in CockroachDB
+			// https://github.com/cockroachdb/cockroach/blob/master/pkg/sql/pgwire/pgerror/codes.go
+			if e.Code == "42P01" {
+				return database.NilVersion, false, nil
+			}
+		}
+		return 0, false, &database.Error{OrigErr: err, Query: []byte(query)}
+
+	default:
+		return version, dirty, nil
+	}
+}
+
+func (c *CockroachDb) Drop() (err error) {
+	// select all tables in current schema
+	query := `SELECT table_name FROM information_schema.tables WHERE table_schema=(SELECT current_schema())`
+	tables, err := c.db.Query(query)
+	if err != nil {
+		return &database.Error{OrigErr: err, Query: []byte(query)}
+	}
+	defer func() {
+		if errClose := tables.Close(); errClose != nil {
+			err = multierror.Append(err, errClose)
+		}
+	}()
+
+	// delete one table after another
+	tableNames := make([]string, 0)
+	for tables.Next() {
+		var tableName string
+		if err := tables.Scan(&tableName); err != nil {
+			return err
+		}
+		if len(tableName) > 0 {
+			tableNames = append(tableNames, tableName)
+		}
+	}
+
+	if len(tableNames) > 0 {
+		// delete one by one ...
+		for _, t := range tableNames {
+			query = `DROP TABLE IF EXISTS ` + t + ` CASCADE`
+			if _, err := c.db.Exec(query); err != nil {
+				return &database.Error{OrigErr: err, Query: []byte(query)}
+			}
+		}
+	}
+
+	return nil
+}
+
+// ensureVersionTable checks if versions table exists and, if not, creates it.
+// Note that this function locks the database, which deviates from the usual
+// convention of "caller locks" in the CockroachDb type.
+func (c *CockroachDb) ensureVersionTable() (err error) {
+	if err = c.Lock(); err != nil {
+		return err
+	}
+
+	defer func() {
+		if e := c.Unlock(); e != nil {
+			if err == nil {
+				err = e
+			} else {
+				err = multierror.Append(err, e)
+			}
+		}
+	}()
+
+	// check if migration table exists
+	var count int
+	query := `SELECT COUNT(1) FROM information_schema.tables WHERE table_name = $1 AND table_schema = (SELECT current_schema()) LIMIT 1`
+	if err := c.db.QueryRow(query, c.config.MigrationsTable).Scan(&count); err != nil {
+		return &database.Error{OrigErr: err, Query: []byte(query)}
+	}
+	if count == 1 {
+		return nil
+	}
+
+	// if not, create the empty migration table
+	query = `CREATE TABLE "` + c.config.MigrationsTable + `" (version INT NOT NULL PRIMARY KEY, dirty BOOL NOT NULL)`
+	if _, err := c.db.Exec(query); err != nil {
+		return &database.Error{OrigErr: err, Query: []byte(query)}
+	}
+	return nil
+}
+
+func (c *CockroachDb) ensureLockTable() error {
+	// check if lock table exists
+	var count int
+	query := `SELECT COUNT(1) FROM information_schema.tables WHERE table_name = $1 AND table_schema = (SELECT current_schema()) LIMIT 1`
+	if err := c.db.QueryRow(query, c.config.LockTable).Scan(&count); err != nil {
+		return &database.Error{OrigErr: err, Query: []byte(query)}
+	}
+	if count == 1 {
+		return nil
+	}
+
+	// if not, create the empty lock table
+	query = `CREATE TABLE "` + c.config.LockTable + `" (lock_id INT NOT NULL PRIMARY KEY)`
+	if _, err := c.db.Exec(query); err != nil {
+		return &database.Error{OrigErr: err, Query: []byte(query)}
+	}
+
+	return nil
+}

database/cockroachdb/cockroachdb_test.go

@@ -0,0 +1,174 @@
+package cockroachdb
+
+// error codes https://github.com/lib/pq/blob/master/error.go
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"github.com/golang-migrate/migrate/v4"
+	"log"
+	"strings"
+	"testing"
+)
+
+import (
+	"github.com/dhui/dktest"
+	_ "github.com/lib/pq"
+)
+
+import (
+	dt "github.com/golang-migrate/migrate/v4/database/testing"
+	"github.com/golang-migrate/migrate/v4/dktesting"
+	_ "github.com/golang-migrate/migrate/v4/source/file"
+)
+
+const defaultPort = 26257
+
+var (
+	opts = dktest.Options{Cmd: []string{"start", "--insecure"}, PortRequired: true, ReadyFunc: isReady}
+	// Released versions: https://www.cockroachlabs.com/docs/releases/
+	specs = []dktesting.ContainerSpec{
+		{ImageName: "cockroachdb/cockroach:v1.0.7", Options: opts},
+		{ImageName: "cockroachdb/cockroach:v1.1.9", Options: opts},
+		{ImageName: "cockroachdb/cockroach:v2.0.7", Options: opts},
+		{ImageName: "cockroachdb/cockroach:v2.1.3", Options: opts},
+	}
+)
+
+func isReady(ctx context.Context, c dktest.ContainerInfo) bool {
+	ip, port, err := c.Port(defaultPort)
+	if err != nil {
+		log.Println("port error:", err)
+		return false
+	}
+
+	db, err := sql.Open("postgres", fmt.Sprintf("postgres://root@%v:%v?sslmode=disable", ip, port))
+	if err != nil {
+		log.Println("open error:", err)
+		return false
+	}
+	if err := db.PingContext(ctx); err != nil {
+		log.Println("ping error:", err)
+		return false
+	}
+	if err := db.Close(); err != nil {
+		log.Println("close error:", err)
+	}
+	return true
+}
+
+func createDB(t *testing.T, c dktest.ContainerInfo) {
+	ip, port, err := c.Port(defaultPort)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	db, err := sql.Open("postgres", fmt.Sprintf("postgres://root@%v:%v?sslmode=disable", ip, port))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err = db.Ping(); err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		if err := db.Close(); err != nil {
+			t.Error(err)
+		}
+	}()
+
+	if _, err = db.Exec("CREATE DATABASE migrate"); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func Test(t *testing.T) {
+	dktesting.ParallelTest(t, specs, func(t *testing.T, ci dktest.ContainerInfo) {
+		createDB(t, ci)
+
+		ip, port, err := ci.Port(26257)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		addr := fmt.Sprintf("cockroach://root@%v:%v/migrate?sslmode=disable", ip, port)
+		c := &CockroachDb{}
+		d, err := c.Open(addr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		dt.Test(t, d, []byte("SELECT 1"))
+	})
+}
+
+func TestMigrate(t *testing.T) {
+	dktesting.ParallelTest(t, specs, func(t *testing.T, ci dktest.ContainerInfo) {
+		createDB(t, ci)
+
+		ip, port, err := ci.Port(26257)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		addr := fmt.Sprintf("cockroach://root@%v:%v/migrate?sslmode=disable", ip, port)
+		c := &CockroachDb{}
+		d, err := c.Open(addr)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		m, err := migrate.NewWithDatabaseInstance("file://./examples/migrations", "migrate", d)
+		if err != nil {
+			t.Fatal(err)
+		}
+		dt.TestMigrate(t, m, []byte("SELECT 1"))
+	})
+}
+
+func TestMultiStatement(t *testing.T) {
+	dktesting.ParallelTest(t, specs, func(t *testing.T, ci dktest.ContainerInfo) {
+		createDB(t, ci)
+
+		ip, port, err := ci.Port(26257)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		addr := fmt.Sprintf("cockroach://root@%v:%v/migrate?sslmode=disable", ip, port)
+		c := &CockroachDb{}
+		d, err := c.Open(addr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if err := d.Run(strings.NewReader("CREATE TABLE foo (foo text); CREATE TABLE bar (bar text);")); err != nil {
+			t.Fatalf("expected err to be nil, got %v", err)
+		}
+
+		// make sure second table exists
+		var exists bool
+		if err := d.(*CockroachDb).db.QueryRow("SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'bar' AND table_schema = (SELECT current_schema()))").Scan(&exists); err != nil {
+			t.Fatal(err)
+		}
+		if !exists {
+			t.Fatalf("expected table bar to exist")
+		}
+	})
+}
+
+func TestFilterCustomQuery(t *testing.T) {
+	dktesting.ParallelTest(t, specs, func(t *testing.T, ci dktest.ContainerInfo) {
+		createDB(t, ci)
+
+		ip, port, err := ci.Port(26257)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		addr := fmt.Sprintf("cockroach://root@%v:%v/migrate?sslmode=disable&x-custom=foobar", ip, port)
+		c := &CockroachDb{}
+		_, err = c.Open(addr)
+		if err != nil {
+			t.Fatal(err)
+		}
+	})
+}

database/cockroachdb/examples/migrations/1085649617_create_users_table.down.sql

@@ -0,0 +1 @@
+DROP TABLE IF EXISTS users;

database/cockroachdb/examples/migrations/1085649617_create_users_table.up.sql

@@ -0,0 +1,5 @@
+CREATE TABLE users (
+  user_id INT UNIQUE,
+  name    STRING(40),
+  email   STRING(40)
+);

database/cockroachdb/examples/migrations/1185749658_add_city_to_users.down.sql

@@ -0,0 +1 @@
+ALTER TABLE users DROP COLUMN IF EXISTS city;

database/cockroachdb/examples/migrations/1185749658_add_city_to_users.up.sql

@@ -0,0 +1 @@
+ALTER TABLE users ADD COLUMN city TEXT;

database/cockroachdb/examples/migrations/1285849751_add_index_on_user_emails.down.sql

@@ -0,0 +1 @@
+DROP INDEX IF EXISTS users_email_index;

database/cockroachdb/examples/migrations/1285849751_add_index_on_user_emails.up.sql

@@ -0,0 +1,3 @@
+CREATE UNIQUE INDEX IF NOT EXISTS users_email_index ON users (email);
+
+-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere.

database/cockroachdb/examples/migrations/1385949617_create_books_table.down.sql

@@ -0,0 +1 @@
+DROP TABLE IF EXISTS books;

database/cockroachdb/examples/migrations/1385949617_create_books_table.up.sql

@@ -0,0 +1,5 @@
+CREATE TABLE books (
+  user_id INT,
+  name    STRING(40),
+  author  STRING(40)
+);

database/cockroachdb/examples/migrations/1485949617_create_movies_table.down.sql

@@ -0,0 +1 @@
+DROP TABLE IF EXISTS movies;

database/cockroachdb/examples/migrations/1485949617_create_movies_table.up.sql

@@ -0,0 +1,5 @@
+CREATE TABLE movies (
+  user_id   INT,
+  name      STRING(40),
+  director  STRING(40)
+);

database/cockroachdb/examples/migrations/1585849751_just_a_comment.up.sql

@@ -0,0 +1 @@
+-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere.

database/cockroachdb/examples/migrations/1685849751_another_comment.up.sql

@@ -0,0 +1 @@
+-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere.

database/cockroachdb/examples/migrations/1785849751_another_comment.up.sql

@@ -0,0 +1 @@
+-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere.

database/cockroachdb/examples/migrations/1885849751_another_comment.up.sql

@@ -0,0 +1 @@
+-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere.

database/driver.go

@@ -7,8 +7,9 @@ package database
 import (
 	"fmt"
 	"io"
-	nurl "net/url"
 	"sync"
+
+	iurl "github.com/golang-migrate/migrate/v4/internal/url"
 )
 
 var (
@@ -32,6 +33,8 @@ var drivers = make(map[string]Driver)
 //      All other functions are tested by tests in database/testing.
 //      Saves you some time and makes sure all database drivers behave the same way.
 //   5. Call Register in init().
+//   6. Create a migrate/cli/build_<driver-name>.go file
+//   7. Add driver name in 'DATABASE' variable in Makefile
 //
 // Guidelines:
 //   * Don't try to correct user input. Don't assume things.
@@ -71,26 +74,24 @@ type Driver interface {
 	// Dirty means, a previous migration failed and user interaction is required.
 	Version() (version int, dirty bool, err error)
 
-	// Drop deletes everyting in the database.
+	// Drop deletes everything in the database.
+	// Note that this is a breaking action, a new call to Open() is necessary to
+	// ensure subsequent calls work as expected.
 	Drop() error
 }
 
 // Open returns a new driver instance.
 func Open(url string) (Driver, error) {
-	u, err := nurl.Parse(url)
+	scheme, err := iurl.SchemeFromURL(url)
 	if err != nil {
 		return nil, err
 	}
 
-	if u.Scheme == "" {
-		return nil, fmt.Errorf("database driver: invalid URL scheme")
-	}
-
 	driversMu.RLock()
-	d, ok := drivers[u.Scheme]
+	d, ok := drivers[scheme]
 	driversMu.RUnlock()
 	if !ok {
-		return nil, fmt.Errorf("database driver: unknown driver %v (forgotton import?)", u.Scheme)
+		return nil, fmt.Errorf("database driver: unknown driver %v (forgotten import?)", scheme)
 	}
 
 	return d.Open(url)
@@ -108,3 +109,14 @@ func Register(name string, driver Driver) {
 	}
 	drivers[name] = driver
 }
+
+// List lists the registered drivers
+func List() []string {
+	driversMu.RLock()
+	defer driversMu.RUnlock()
+	names := make([]string, 0, len(drivers))
+	for n := range drivers {
+		names = append(names, n)
+	}
+	return names
+}

database/driver_test.go

@@ -1,8 +1,115 @@
 package database
 
+import (
+	"io"
+	"testing"
+)
+
 func ExampleDriver() {
 	// see database/stub for an example
 
 	// database/stub/stub.go has the driver implementation
 	// database/stub/stub_test.go runs database/testing/test.go:Test
 }
+
+// Using database/stub here is not possible as it
+// results in an import cycle.
+type mockDriver struct {
+	url string
+}
+
+func (m *mockDriver) Open(url string) (Driver, error) {
+	return &mockDriver{
+		url: url,
+	}, nil
+}
+
+func (m *mockDriver) Close() error {
+	return nil
+}
+
+func (m *mockDriver) Lock() error {
+	return nil
+}
+
+func (m *mockDriver) Unlock() error {
+	return nil
+}
+
+func (m *mockDriver) Run(migration io.Reader) error {
+	return nil
+}
+
+func (m *mockDriver) SetVersion(version int, dirty bool) error {
+	return nil
+}
+
+func (m *mockDriver) Version() (version int, dirty bool, err error) {
+	return 0, false, nil
+}
+
+func (m *mockDriver) Drop() error {
+	return nil
+}
+
+func TestRegisterTwice(t *testing.T) {
+	Register("mock", &mockDriver{})
+
+	var err interface{}
+	func() {
+		defer func() {
+			err = recover()
+		}()
+		Register("mock", &mockDriver{})
+	}()
+
+	if err == nil {
+		t.Fatal("expected a panic when calling Register twice")
+	}
+}
+
+func TestOpen(t *testing.T) {
+	// Make sure the driver is registered.
+	// But if the previous test already registered it just ignore the panic.
+	// If we don't do this it will be impossible to run this test standalone.
+	func() {
+		defer func() {
+			_ = recover()
+		}()
+		Register("mock", &mockDriver{})
+	}()
+
+	cases := []struct {
+		url string
+		err bool
+	}{
+		{
+			"mock://user:pass@tcp(host:1337)/db",
+			false,
+		},
+		{
+			"unknown://bla",
+			true,
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.url, func(t *testing.T) {
+			d, err := Open(c.url)
+
+			if err == nil {
+				if c.err {
+					t.Fatal("expected an error for an unknown driver")
+				} else {
+					if md, ok := d.(*mockDriver); !ok {
+						t.Fatalf("expected *mockDriver got %T", d)
+					} else if md.url != c.url {
+						t.Fatalf("expected %q got %q", c.url, md.url)
+					}
+				}
+			} else if !c.err {
+				t.Fatalf("did not expect %q", err)
+			}
+		})
+	}
+}

database/firebird/README.md

@@ -0,0 +1,12 @@
+# firebird
+
+`firebirdsql://user:password@servername[:port_number]/database_name_or_file[?params1=value1[&param2=value2]...]`
+
+| URL Query  | WithInstance Config | Description |
+|------------|---------------------|-------------|
+| `x-migrations-table` | `MigrationsTable` | Name of the migrations table |
+| `auth_plugin_name` | | Authentication plugin name. Srp256/Srp/Legacy_Auth are available. (default is Srp) |
+| `column_name_to_lower` | | Force column name to lower. (default is false) |
+| `role` | | Role name |
+| `tzname` | | Time Zone name. (For Firebird 4.0+) |
+| `wire_crypt` | | Enable wire data encryption or not. For Firebird 3.0+ (default is true) |

database/firebird/examples/migrations/1085649617_create_users_table.down.sql

@@ -0,0 +1 @@
+DROP TABLE users;

database/firebird/examples/migrations/1085649617_create_users_table.up.sql

@@ -0,0 +1,5 @@
+CREATE TABLE users (
+  user_id integer unique,
+  name    varchar(40),
+  email   varchar(40)
+);

database/firebird/examples/migrations/1185749658_add_city_to_users.down.sql

@@ -0,0 +1 @@
+ALTER TABLE users DROP city;

database/firebird/examples/migrations/1185749658_add_city_to_users.up.sql

@@ -0,0 +1,3 @@
+ALTER TABLE users ADD city varchar(100);
+
+

database/firebird/examples/migrations/1285849751_add_index_on_user_emails.down.sql

@@ -0,0 +1 @@
+DROP INDEX users_email_index;

database/firebird/examples/migrations/1285849751_add_index_on_user_emails.up.sql

@@ -0,0 +1,3 @@
+CREATE UNIQUE INDEX users_email_index ON users (email);
+
+-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere.

database/firebird/examples/migrations/1385949617_create_books_table.down.sql

@@ -0,0 +1 @@
+DROP TABLE books;

database/firebird/examples/migrations/1385949617_create_books_table.up.sql

@@ -0,0 +1,5 @@
+CREATE TABLE books (
+  user_id integer,
+  name    varchar(40),
+  author  varchar(40)
+);

database/firebird/examples/migrations/1485949617_create_movies_table.down.sql

@@ -0,0 +1 @@
+DROP TABLE movies;

database/firebird/examples/migrations/1485949617_create_movies_table.up.sql

@@ -0,0 +1,5 @@
+CREATE TABLE movies (
+  user_id   integer,
+  name      varchar(40),
+  director  varchar(40)
+);

database/firebird/firebird.go

@@ -0,0 +1,250 @@
+// +build go1.9
+
+package firebird
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"github.com/golang-migrate/migrate/v4"
+	"github.com/golang-migrate/migrate/v4/database"
+	"github.com/hashicorp/go-multierror"
+	_ "github.com/nakagami/firebirdsql"
+	"io"
+	"io/ioutil"
+	nurl "net/url"
+)
+
+func init() {
+	db := Firebird{}
+	database.Register("firebird", &db)
+	database.Register("firebirdsql", &db)
+}
+
+var DefaultMigrationsTable = "schema_migrations"
+
+var (
+	ErrNilConfig = fmt.Errorf("no config")
+)
+
+type Config struct {
+	DatabaseName    string
+	MigrationsTable string
+}
+
+type Firebird struct {
+	// Locking and unlocking need to use the same connection
+	conn     *sql.Conn
+	db       *sql.DB
+	isLocked bool
+
+	// Open and WithInstance need to guarantee that config is never nil
+	config *Config
+}
+
+func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) {
+	if config == nil {
+		return nil, ErrNilConfig
+	}
+
+	if err := instance.Ping(); err != nil {
+		return nil, err
+	}
+
+	if len(config.MigrationsTable) == 0 {
+		config.MigrationsTable = DefaultMigrationsTable
+	}
+
+	conn, err := instance.Conn(context.Background())
+	if err != nil {
+		return nil, err
+	}
+
+	fb := &Firebird{
+		conn:   conn,
+		db:     instance,
+		config: config,
+	}
+
+	if err := fb.ensureVersionTable(); err != nil {
+		return nil, err
+	}
+
+	return fb, nil
+}
+
+func (f *Firebird) Open(dsn string) (database.Driver, error) {
+	purl, err := nurl.Parse(dsn)
+	if err != nil {
+		return nil, err
+	}
+
+	db, err := sql.Open("firebirdsql", migrate.FilterCustomQuery(purl).String())
+	if err != nil {
+		return nil, err
+	}
+
+	px, err := WithInstance(db, &Config{
+		MigrationsTable: purl.Query().Get("x-migrations-table"),
+		DatabaseName:    purl.Path,
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return px, nil
+}
+
+func (f *Firebird) Close() error {
+	connErr := f.conn.Close()
+	dbErr := f.db.Close()
+	if connErr != nil || dbErr != nil {
+		return fmt.Errorf("conn: %v, db: %v", connErr, dbErr)
+	}
+	return nil
+}
+
+func (f *Firebird) Lock() error {
+	if f.isLocked {
+		return database.ErrLocked
+	}
+	f.isLocked = true
+	return nil
+}
+
+func (f *Firebird) Unlock() error {
+	f.isLocked = false
+	return nil
+}
+
+func (f *Firebird) Run(migration io.Reader) error {
+	migr, err := ioutil.ReadAll(migration)
+	if err != nil {
+		return err
+	}
+
+	// run migration
+	query := string(migr[:])
+	if _, err := f.conn.ExecContext(context.Background(), query); err != nil {
+		return database.Error{OrigErr: err, Err: "migration failed", Query: migr}
+	}
+
+	return nil
+}
+
+func (f *Firebird) SetVersion(version int, dirty bool) error {
+	if version < 0 {
+		return nil
+	}
+
+	query := fmt.Sprintf(`EXECUTE BLOCK AS BEGIN
+					DELETE FROM "%v";
+					INSERT INTO "%v" (version, dirty) VALUES (%v, %v);
+				END;`,
+		f.config.MigrationsTable, f.config.MigrationsTable, version, btoi(dirty))
+
+	if _, err := f.conn.ExecContext(context.Background(), query, version, btoi(dirty)); err != nil {
+		return &database.Error{OrigErr: err, Query: []byte(query)}
+	}
+
+	return nil
+}
+
+func (f *Firebird) Version() (version int, dirty bool, err error) {
+	var d int
+	query := fmt.Sprintf(`SELECT FIRST 1 version, dirty FROM "%v"`, f.config.MigrationsTable)
+	err = f.conn.QueryRowContext(context.Background(), query).Scan(&version, &d)
+	switch {
+	case err == sql.ErrNoRows:
+		return database.NilVersion, false, nil
+	case err != nil:
+		return 0, false, &database.Error{OrigErr: err, Query: []byte(query)}
+
+	default:
+		return version, itob(d), nil
+	}
+}
+
+func (f *Firebird) Drop() (err error) {
+	// select all tables
+	query := `SELECT rdb$relation_name FROM rdb$relations WHERE rdb$view_blr IS NULL AND (rdb$system_flag IS NULL OR rdb$system_flag = 0);`
+	tables, err := f.conn.QueryContext(context.Background(), query)
+	if err != nil {
+		return &database.Error{OrigErr: err, Query: []byte(query)}
+	}
+	defer func() {
+		if errClose := tables.Close(); errClose != nil {
+			err = multierror.Append(err, errClose)
+		}
+	}()
+
+	// delete one table after another
+	tableNames := make([]string, 0)
+	for tables.Next() {
+		var tableName string
+		if err := tables.Scan(&tableName); err != nil {
+			return err
+		}
+		if len(tableName) > 0 {
+			tableNames = append(tableNames, tableName)
+		}
+	}
+
+	// delete one by one ...
+	for _, t := range tableNames {
+		query := fmt.Sprintf(`EXECUTE BLOCK AS BEGIN
+						if (not exists(select 1 from rdb$relations where rdb$relation_name = '%v')) then
+						execute statement 'drop table "%v"';
+					END;`,
+			t, t)
+
+		if _, err := f.conn.ExecContext(context.Background(), query); err != nil {
+			return &database.Error{OrigErr: err, Query: []byte(query)}
+		}
+	}
+
+	return nil
+}
+
+// ensureVersionTable checks if versions table exists and, if not, creates it.
+func (f *Firebird) ensureVersionTable() (err error) {
+	if err = f.Lock(); err != nil {
+		return err
+	}
+
+	defer func() {
+		if e := f.Unlock(); e != nil {
+			if err == nil {
+				err = e
+			} else {
+				err = multierror.Append(err, e)
+			}
+		}
+	}()
+
+	query := fmt.Sprintf(`EXECUTE BLOCK AS BEGIN
+			if (not exists(select 1 from rdb$relations where rdb$relation_name = '%v')) then
+			execute statement 'create table "%v" (version bigint not null primary key, dirty smallint not null)';
+		END;`,
+		f.config.MigrationsTable, f.config.MigrationsTable)
+
+	if _, err = f.conn.ExecContext(context.Background(), query); err != nil {
+		return &database.Error{OrigErr: err, Query: []byte(query)}
+	}
+
+	return nil
+}
+
+// btoi converts bool to int
+func btoi(v bool) int {
+	if v {
+		return 1
+	}
+	return 0
+}
+
+// itob converts int to bool
+func itob(v int) bool {
+	return v != 0
+}

database/firebird/firebird_test.go

@@ -0,0 +1,226 @@
+package firebird
+
+import (
+	"context"
+	"database/sql"
+	sqldriver "database/sql/driver"
+	"fmt"
+	"log"
+
+	"github.com/golang-migrate/migrate/v4"
+	"io"
+	"strings"
+	"testing"
+
+	"github.com/dhui/dktest"
+
+	dt "github.com/golang-migrate/migrate/v4/database/testing"
+	"github.com/golang-migrate/migrate/v4/dktesting"
+	_ "github.com/golang-migrate/migrate/v4/source/file"
+
+	_ "github.com/nakagami/firebirdsql"
+)
+
+const (
+	user     = "test_user"
+	password = "123456"
+	dbName   = "test.fdb"
+)
+
+var (
+	opts = dktest.Options{
+		PortRequired: true,
+		ReadyFunc:    isReady,
+		Env: map[string]string{
+			"FIREBIRD_DATABASE": dbName,
+			"FIREBIRD_USER":     user,
+			"FIREBIRD_PASSWORD": password,
+		},
+	}
+	specs = []dktesting.ContainerSpec{
+		{ImageName: "jacobalberty/firebird:2.5-ss", Options: opts},
+		{ImageName: "jacobalberty/firebird:3.0", Options: opts},
+	}
+)
+
+func fbConnectionString(host, port string) string {
+	//firebird://user:password@servername[:port_number]/database_name_or_file[?params1=value1[&param2=value2]...]
+	return fmt.Sprintf("firebird://%s:%s@%s:%s//firebird/data/%s", user, password, host, port, dbName)
+}
+
+func isReady(ctx context.Context, c dktest.ContainerInfo) bool {
+	ip, port, err := c.FirstPort()
+	if err != nil {
+		return false
+	}
+
+	db, err := sql.Open("firebirdsql", fbConnectionString(ip, port))
+	if err != nil {
+		log.Println("open error:", err)
+		return false
+	}
+	defer func() {
+		if err := db.Close(); err != nil {
+			log.Println("close error:", err)
+		}
+	}()
+	if err = db.PingContext(ctx); err != nil {
+		switch err {
+		case sqldriver.ErrBadConn, io.EOF:
+			return false
+		default:
+			log.Println(err)
+		}
+		return false
+	}
+
+	return true
+}
+
+func Test(t *testing.T) {
+	dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
+		ip, port, err := c.FirstPort()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		addr := fbConnectionString(ip, port)
+		p := &Firebird{}
+		d, err := p.Open(addr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer func() {
+			if err := d.Close(); err != nil {
+				t.Error(err)
+			}
+		}()
+		dt.Test(t, d, []byte("SELECT Count(*) FROM rdb$relations"))
+	})
+}
+
+func TestMigrate(t *testing.T) {
+	dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
+		ip, port, err := c.FirstPort()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		addr := fbConnectionString(ip, port)
+		p := &Firebird{}
+		d, err := p.Open(addr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer func() {
+			if err := d.Close(); err != nil {
+				t.Error(err)
+			}
+		}()
+		m, err := migrate.NewWithDatabaseInstance("file://./examples/migrations", "firebirdsql", d)
+		if err != nil {
+			t.Fatal(err)
+		}
+		dt.TestMigrate(t, m, []byte("SELECT Count(*) FROM rdb$relations"))
+	})
+}
+
+func TestErrorParsing(t *testing.T) {
+	dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
+		ip, port, err := c.FirstPort()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		addr := fbConnectionString(ip, port)
+		p := &Firebird{}
+		d, err := p.Open(addr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer func() {
+			if err := d.Close(); err != nil {
+				t.Error(err)
+			}
+		}()
+
+		wantErr := `migration failed in line 0: CREATE TABLEE foo (foo varchar(40)); (details: Dynamic SQL Error
+SQL error code = -104
+Token unknown - line 1, column 8
+TABLEE
+)`
+
+		if err := d.Run(strings.NewReader("CREATE TABLEE foo (foo varchar(40));")); err == nil {
+			t.Fatal("expected err but got nil")
+		} else if err.Error() != wantErr {
+			msg := err.Error()
+			t.Fatalf("expected '%s' but got '%s'", wantErr, msg)
+		}
+	})
+}
+
+func TestFilterCustomQuery(t *testing.T) {
+	dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
+		ip, port, err := c.FirstPort()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		addr := fbConnectionString(ip, port) + "?sslmode=disable&x-custom=foobar"
+		p := &Firebird{}
+		d, err := p.Open(addr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer func() {
+			if err := d.Close(); err != nil {
+				t.Error(err)
+			}
+		}()
+	})
+}
+
+func Test_Lock(t *testing.T) {
+	dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
+		ip, port, err := c.FirstPort()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		addr := fbConnectionString(ip, port)
+		p := &Firebird{}
+		d, err := p.Open(addr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer func() {
+			if err := d.Close(); err != nil {
+				t.Error(err)
+			}
+		}()
+
+		dt.Test(t, d, []byte("SELECT Count(*) FROM rdb$relations"))
+
+		ps := d.(*Firebird)
+
+		err = ps.Lock()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		err = ps.Unlock()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		err = ps.Lock()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		err = ps.Unlock()
+		if err != nil {
+			t.Fatal(err)
+		}
+	})
+}

database/mongodb/README.md

@@ -0,0 +1,20 @@
+# MongoDB
+
+* Driver work with mongo through [db.runCommands](https://docs.mongodb.com/manual/reference/command/)
+* Migrations support json format. It contains array of commands for `db.runCommand`. Every command is executed in separate request to database 
+* All keys have to be in quotes `"`
+* [Examples](./examples)
+
+# Usage
+
+`mongodb://user:password@host:port/dbname?query`
+
+| URL Query  | WithInstance Config | Description |
+|------------|---------------------|-------------|
+| `x-migrations-collection` | `MigrationsCollection` | Name of the migrations collection |
+| `x-transaction-mode` | `TransactionMode` | If set to `true` wrap commands in [transaction](https://docs.mongodb.com/manual/core/transactions). Available only for replica set. Driver is using [strconv.ParseBool](https://golang.org/pkg/strconv/#ParseBool) for parsing|
+| `dbname` | `DatabaseName` | The name of the database to connect to |
+| `user` | | The user to sign in as. Can be omitted |
+| `password` | | The user's password. Can be omitted | 
+| `host` | | The host to connect to |
+| `port` | | The port to bind to |

database/mongodb/examples/migrations/001_create_user.down.json

@@ -0,0 +1,5 @@
+[
+  {
+    "dropUser": "deminem"
+  }
+]

database/mongodb/examples/migrations/001_create_user.up.json

@@ -0,0 +1,12 @@
+[
+  {
+    "createUser": "deminem",
+    "pwd": "gogo",
+    "roles": [
+      {
+        "role": "readWrite",
+        "db": "testMigration"
+      }
+    ]
+  }
+]

database/mongodb/examples/migrations/002_create_indexes.down.json

@@ -0,0 +1,10 @@
+[
+  {
+    "dropIndexes": "mycollection",
+    "index": "username_sort_by_asc_created"
+  },
+  {
+    "dropIndexes": "mycollection",
+    "index": "unique_email"
+  }
+]

database/mongodb/examples/migrations/002_create_indexes.up.json

@@ -0,0 +1,21 @@
+[{
+  "createIndexes": "mycollection",
+  "indexes": [
+    {
+      "key": {
+        "username": 1,
+        "created": -1
+      },
+      "name": "username_sort_by_asc_created",
+      "background": true
+    },
+    {
+      "key": {
+        "email": 1
+      },
+      "name": "unique_email",
+      "unique": true,
+      "background": true
+    }
+  ]
+}]

database/mongodb/mongodb.go

@@ -0,0 +1,191 @@
+package mongodb
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/url"
+	"strconv"
+
+	"github.com/golang-migrate/migrate/v4/database"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+	"go.mongodb.org/mongo-driver/x/mongo/driver/connstring"
+)
+
+func init() {
+	database.Register("mongodb", &Mongo{})
+}
+
+var DefaultMigrationsCollection = "schema_migrations"
+
+var (
+	ErrNoDatabaseName = fmt.Errorf("no database name")
+	ErrNilConfig      = fmt.Errorf("no config")
+)
+
+type Mongo struct {
+	client *mongo.Client
+	db     *mongo.Database
+
+	config *Config
+}
+
+type Config struct {
+	DatabaseName         string
+	MigrationsCollection string
+	TransactionMode      bool
+}
+
+type versionInfo struct {
+	Version int  `bson:"version"`
+	Dirty   bool `bson:"dirty"`
+}
+
+func WithInstance(instance *mongo.Client, config *Config) (database.Driver, error) {
+	if config == nil {
+		return nil, ErrNilConfig
+	}
+	if len(config.DatabaseName) == 0 {
+		return nil, ErrNoDatabaseName
+	}
+	if len(config.MigrationsCollection) == 0 {
+		config.MigrationsCollection = DefaultMigrationsCollection
+	}
+	mc := &Mongo{
+		client: instance,
+		db:     instance.Database(config.DatabaseName),
+		config: config,
+	}
+
+	return mc, nil
+}
+
+func (m *Mongo) Open(dsn string) (database.Driver, error) {
+	//connsting is experimental package, but it used for parse connection string in mongo.Connect function
+	uri, err := connstring.Parse(dsn)
+	if err != nil {
+		return nil, err
+	}
+	if len(uri.Database) == 0 {
+		return nil, ErrNoDatabaseName
+	}
+
+	unknown := url.Values(uri.UnknownOptions)
+
+	migrationsCollection := unknown.Get("x-migrations-collection")
+	transactionMode, _ := strconv.ParseBool(unknown.Get("x-transaction-mode"))
+
+	client, err := mongo.Connect(context.TODO(), options.Client().ApplyURI(dsn))
+	if err != nil {
+		return nil, err
+	}
+	if err = client.Ping(context.TODO(), nil); err != nil {
+		return nil, err
+	}
+	mc, err := WithInstance(client, &Config{
+		DatabaseName:         uri.Database,
+		MigrationsCollection: migrationsCollection,
+		TransactionMode:      transactionMode,
+	})
+	if err != nil {
+		return nil, err
+	}
+	return mc, nil
+}
+
+func (m *Mongo) SetVersion(version int, dirty bool) error {
+	migrationsCollection := m.db.Collection(m.config.MigrationsCollection)
+	if err := migrationsCollection.Drop(context.TODO()); err != nil {
+		return &database.Error{OrigErr: err, Err: "drop migrations collection failed"}
+	}
+	_, err := migrationsCollection.InsertOne(context.TODO(), bson.M{"version": version, "dirty": dirty})
+	if err != nil {
+		return &database.Error{OrigErr: err, Err: "save version failed"}
+	}
+	return nil
+}
+
+func (m *Mongo) Version() (version int, dirty bool, err error) {
+	var versionInfo versionInfo
+	err = m.db.Collection(m.config.MigrationsCollection).FindOne(context.TODO(), bson.M{}).Decode(&versionInfo)
+	switch {
+	case err == mongo.ErrNoDocuments:
+		return database.NilVersion, false, nil
+	case err != nil:
+		return 0, false, &database.Error{OrigErr: err, Err: "failed to get migration version"}
+	default:
+		return versionInfo.Version, versionInfo.Dirty, nil
+	}
+}
+
+func (m *Mongo) Run(migration io.Reader) error {
+	migr, err := ioutil.ReadAll(migration)
+	if err != nil {
+		return err
+	}
+	var cmds []bson.D
+	err = bson.UnmarshalExtJSON(migr, true, &cmds)
+	if err != nil {
+		return fmt.Errorf("unmarshaling json error: %s", err)
+	}
+	if m.config.TransactionMode {
+		if err := m.executeCommandsWithTransaction(context.TODO(), cmds); err != nil {
+			return err
+		}
+	} else {
+		if err := m.executeCommands(context.TODO(), cmds); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (m *Mongo) executeCommandsWithTransaction(ctx context.Context, cmds []bson.D) error {
+	err := m.db.Client().UseSession(ctx, func(sessionContext mongo.SessionContext) error {
+		if err := sessionContext.StartTransaction(); err != nil {
+			return &database.Error{OrigErr: err, Err: "failed to start transaction"}
+		}
+		if err := m.executeCommands(sessionContext, cmds); err != nil {
+			//When command execution is failed, it's aborting transaction
+			//If you tried to call abortTransaction, it`s return error that transaction already aborted
+			return err
+		}
+		if err := sessionContext.CommitTransaction(sessionContext); err != nil {
+			return &database.Error{OrigErr: err, Err: "failed to commit transaction"}
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (m *Mongo) executeCommands(ctx context.Context, cmds []bson.D) error {
+	for _, cmd := range cmds {
+		err := m.db.RunCommand(ctx, cmd).Err()
+		if err != nil {
+			return &database.Error{OrigErr: err, Err: fmt.Sprintf("failed to execute command:%v", cmd)}
+		}
+	}
+	return nil
+}
+
+func (m *Mongo) Close() error {
+	return m.client.Disconnect(context.TODO())
+}
+
+func (m *Mongo) Drop() error {
+	return m.db.Drop(context.TODO())
+}
+
+func (m *Mongo) Lock() error {
+	return nil
+}
+
+func (m *Mongo) Unlock() error {
+	return nil
+}

database/mongodb/mongodb_test.go

@@ -0,0 +1,349 @@
+package mongodb
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+
+	"log"
+
+	"github.com/golang-migrate/migrate/v4"
+	"io"
+	"os"
+	"strconv"
+	"testing"
+	"time"
+)
+
+import (
+	"github.com/dhui/dktest"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+)
+
+import (
+	dt "github.com/golang-migrate/migrate/v4/database/testing"
+	"github.com/golang-migrate/migrate/v4/dktesting"
+	_ "github.com/golang-migrate/migrate/v4/source/file"
+)
+
+var (
+	opts = dktest.Options{PortRequired: true, ReadyFunc: isReady}
+	// Supported versions: https://www.mongodb.com/support-policy
+	specs = []dktesting.ContainerSpec{
+		{ImageName: "mongo:3.4", Options: opts},
+		{ImageName: "mongo:3.6", Options: opts},
+		{ImageName: "mongo:4.0", Options: opts},
+		{ImageName: "mongo:4.2", Options: opts},
+	}
+)
+
+func mongoConnectionString(host, port string) string {
+	// there is connect option for excluding serverConnection algorithm
+	// it's let avoid errors with mongo replica set connection in docker container
+	return fmt.Sprintf("mongodb://%s:%s/testMigration?connect=direct", host, port)
+}
+
+func isReady(ctx context.Context, c dktest.ContainerInfo) bool {
+	ip, port, err := c.FirstPort()
+	if err != nil {
+		return false
+	}
+
+	client, err := mongo.Connect(ctx, options.Client().ApplyURI(mongoConnectionString(ip, port)))
+	if err != nil {
+		return false
+	}
+	defer func() {
+		if err := client.Disconnect(ctx); err != nil {
+			log.Println("close error:", err)
+		}
+	}()
+
+	if err = client.Ping(ctx, nil); err != nil {
+		switch err {
+		case io.EOF:
+			return false
+		default:
+			log.Println(err)
+		}
+		return false
+	}
+	return true
+}
+
+func Test(t *testing.T) {
+	dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
+		ip, port, err := c.FirstPort()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		addr := mongoConnectionString(ip, port)
+		p := &Mongo{}
+		d, err := p.Open(addr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer func() {
+			if err := d.Close(); err != nil {
+				t.Error(err)
+			}
+		}()
+		dt.TestNilVersion(t, d)
+		//TestLockAndUnlock(t, d) driver doesn't support lock on database level
+		dt.TestRun(t, d, bytes.NewReader([]byte(`[{"insert":"hello","documents":[{"wild":"world"}]}]`)))
+		dt.TestSetVersion(t, d)
+		dt.TestDrop(t, d)
+	})
+}
+
+func TestMigrate(t *testing.T) {
+	dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
+		ip, port, err := c.FirstPort()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		addr := mongoConnectionString(ip, port)
+		p := &Mongo{}
+		d, err := p.Open(addr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer func() {
+			if err := d.Close(); err != nil {
+				t.Error(err)
+			}
+		}()
+		m, err := migrate.NewWithDatabaseInstance("file://./examples/migrations", "", d)
+		if err != nil {
+			t.Fatal(err)
+		}
+		dt.TestMigrate(t, m, []byte(`[{"insert":"hello","documents":[{"wild":"world"}]}]`))
+	})
+}
+
+func TestWithAuth(t *testing.T) {
+	dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
+		ip, port, err := c.FirstPort()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		addr := mongoConnectionString(ip, port)
+		p := &Mongo{}
+		d, err := p.Open(addr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer func() {
+			if err := d.Close(); err != nil {
+				t.Error(err)
+			}
+		}()
+		createUserCMD := []byte(`[{"createUser":"deminem","pwd":"gogo","roles":[{"role":"readWrite","db":"testMigration"}]}]`)
+		err = d.Run(bytes.NewReader(createUserCMD))
+		if err != nil {
+			t.Fatal(err)
+		}
+		testcases := []struct {
+			name            string
+			connectUri      string
+			isErrorExpected bool
+		}{
+			{"right auth data", "mongodb://deminem:gogo@%s:%v/testMigration", false},
+			{"wrong auth data", "mongodb://wrong:auth@%s:%v/testMigration", true},
+		}
+
+		for _, tcase := range testcases {
+			t.Run(tcase.name, func(t *testing.T) {
+				mc := &Mongo{}
+				d, err := mc.Open(fmt.Sprintf(tcase.connectUri, ip, port))
+				if err == nil {
+					defer func() {
+						if err := d.Close(); err != nil {
+							t.Error(err)
+						}
+					}()
+				}
+
+				switch {
+				case tcase.isErrorExpected && err == nil:
+					t.Fatalf("no error when expected")
+				case !tcase.isErrorExpected && err != nil:
+					t.Fatalf("unexpected error: %v", err)
+				}
+			})
+		}
+	})
+}
+
+func TestTransaction(t *testing.T) {
+	transactionSpecs := []dktesting.ContainerSpec{
+		{ImageName: "mongo:4", Options: dktest.Options{PortRequired: true, ReadyFunc: isReady,
+			Cmd: []string{"mongod", "--bind_ip_all", "--replSet", "rs0"}}},
+	}
+	dktesting.ParallelTest(t, transactionSpecs, func(t *testing.T, c dktest.ContainerInfo) {
+		ip, port, err := c.FirstPort()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		client, err := mongo.Connect(context.TODO(), options.Client().ApplyURI(mongoConnectionString(ip, port)))
+		if err != nil {
+			t.Fatal(err)
+		}
+		err = client.Ping(context.TODO(), nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		//rs.initiate()
+		err = client.Database("admin").RunCommand(context.TODO(), bson.D{bson.E{Key: "replSetInitiate", Value: bson.D{}}}).Err()
+		if err != nil {
+			t.Fatal(err)
+		}
+		err = waitForReplicaInit(client)
+		if err != nil {
+			t.Fatal(err)
+		}
+		d, err := WithInstance(client, &Config{
+			DatabaseName: "testMigration",
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer func() {
+			if err := d.Close(); err != nil {
+				t.Error(err)
+			}
+		}()
+		//We have to create collection
+		//transactions don't support operations with creating new dbs, collections
+		//Unique index need for checking transaction aborting
+		insertCMD := []byte(`[
+				{"create":"hello"},
+				{"createIndexes": "hello",
+					"indexes": [{
+						"key": {
+							"wild": 1
+						},
+						"name": "unique_wild",
+						"unique": true,
+						"background": true
+					}]
+			}]`)
+		err = d.Run(bytes.NewReader(insertCMD))
+		if err != nil {
+			t.Fatal(err)
+		}
+		testcases := []struct {
+			name            string
+			cmds            []byte
+			documentsCount  int64
+			isErrorExpected bool
+		}{
+			{
+				name: "success transaction",
+				cmds: []byte(`[{"insert":"hello","documents":[
+										{"wild":"world"},
+										{"wild":"west"},
+										{"wild":"natural"}
+									 ]
+								  }]`),
+				documentsCount:  3,
+				isErrorExpected: false,
+			},
+			{
+				name: "failure transaction",
+				//transaction have to be failure - duplicate unique key wild:west
+				//none of the documents should be added
+				cmds: []byte(`[{"insert":"hello","documents":[{"wild":"flower"}]},
+									{"insert":"hello","documents":[
+										{"wild":"cat"},
+										{"wild":"west"}
+									 ]
+								  }]`),
+				documentsCount:  3,
+				isErrorExpected: true,
+			},
+		}
+		for _, tcase := range testcases {
+			t.Run(tcase.name, func(t *testing.T) {
+				client, err := mongo.Connect(context.TODO(), options.Client().ApplyURI(mongoConnectionString(ip, port)))
+				if err != nil {
+					t.Fatal(err)
+				}
+				err = client.Ping(context.TODO(), nil)
+				if err != nil {
+					t.Fatal(err)
+				}
+				d, err := WithInstance(client, &Config{
+					DatabaseName:    "testMigration",
+					TransactionMode: true,
+				})
+				if err != nil {
+					t.Fatal(err)
+				}
+				defer func() {
+					if err := d.Close(); err != nil {
+						t.Error(err)
+					}
+				}()
+				runErr := d.Run(bytes.NewReader(tcase.cmds))
+				if runErr != nil {
+					if !tcase.isErrorExpected {
+						t.Fatal(runErr)
+					}
+				}
+				documentsCount, err := client.Database("testMigration").Collection("hello").CountDocuments(context.TODO(), bson.M{})
+				if err != nil {
+					t.Fatal(err)
+				}
+				if tcase.documentsCount != documentsCount {
+					t.Fatalf("expected %d and actual %d documents count not equal. run migration error:%s", tcase.documentsCount, documentsCount, runErr)
+				}
+			})
+		}
+	})
+}
+
+type isMaster struct {
+	IsMaster bool `bson:"ismaster"`
+}
+
+func waitForReplicaInit(client *mongo.Client) error {
+	ticker := time.NewTicker(time.Second * 1)
+	defer ticker.Stop()
+	timeout, err := strconv.Atoi(os.Getenv("MIGRATE_TEST_MONGO_REPLICA_SET_INIT_TIMEOUT"))
+	if err != nil {
+		timeout = 30
+	}
+	timeoutTimer := time.NewTimer(time.Duration(timeout) * time.Second)
+	defer timeoutTimer.Stop()
+	for {
+		select {
+		case <-ticker.C:
+			var status isMaster
+			//Check that node is primary because
+			//during replica set initialization, the first node first becomes a secondary and then becomes the primary
+			//should consider that initialization is completed only after the node has become the primary
+			result := client.Database("admin").RunCommand(context.TODO(), bson.D{bson.E{Key: "isMaster", Value: 1}})
+			r, err := result.DecodeBytes()
+			if err != nil {
+				return err
+			}
+			err = bson.Unmarshal(r, &status)
+			if err != nil {
+				return err
+			}
+			if status.IsMaster {
+				return nil
+			}
+		case <-timeoutTimer.C:
+			return fmt.Errorf("replica init timeout")
+		}
+	}
+
+}

database/mysql/README.md

@@ -1,24 +1,54 @@
-# mysql
-
-`mysql://user:password@tcp(host:port)/dbname?query`
-
-| URL Query  | WithInstance Config | Description |
-|------------|---------------------|-------------|
-| `x-migrations-table` | `MigrationsTable` | Name of the migrations table |
-| `dbname` | `DatabaseName` | The name of the database to connect to |
-| `user` | | The user to sign in as |
-| `password` | | The user's password | 
-| `host` | | The host to connect to. |
-| `port` | | The port to bind to. |
-| `x-tls-ca` | | The location of the root certificate file. |
-| `x-tls-cert` | | Cert file location. |
-| `x-tls-key` | | Key file location. | 
-| `x-tls-insecure-skip-verify` | | Whether or not to use SSL (true\|false) | 
-
-## Upgrading from v1
-
-1. Write down the current migration version from schema_migrations
-1. `DROP TABLE schema_migrations`
-2. Wrap your existing migrations in transactions ([BEGIN/COMMIT](https://dev.mysql.com/doc/refman/5.7/en/commit.html)) if you use multiple statements within one migration.
-3. Download and install the latest migrate version.
-4. Force the current migration version with `migrate force <current_version>`.
+# MySQL
+
+`mysql://user:password@tcp(host:port)/dbname?query`
+
+| URL Query  | WithInstance Config | Description |
+|------------|---------------------|-------------|
+| `x-migrations-table` | `MigrationsTable` | Name of the migrations table |
+| `dbname` | `DatabaseName` | The name of the database to connect to |
+| `user` | | The user to sign in as |
+| `password` | | The user's password | 
+| `host` | | The host to connect to. |
+| `port` | | The port to bind to. |
+| `tls`  | | TLS / SSL encrypted connection parameter; see [go-sql-driver](https://github.com/go-sql-driver/mysql#tls). Use any name (e.g. `migrate`) if you want to use a custom TLS config (`x-tls-` queries). |
+| `x-tls-ca` | | The location of the CA (certificate authority) file. |
+| `x-tls-cert` | | The location of the client certicicate file. Must be used with `x-tls-key`. |
+| `x-tls-key` | | The location of the private key file. Must be used with `x-tls-cert`. |
+| `x-tls-insecure-skip-verify` | | Whether or not to use SSL (true\|false) | 
+
+## Use with existing client
+
+If you use the MySQL driver with existing database client, you must create the client with parameter `multiStatements=true`:
+
+```go
+package main
+
+import (
+    "database/sql"
+    
+    _ "github.com/go-sql-driver/mysql"
+    "github.com/golang-migrate/migrate"
+    "github.com/golang-migrate/migrate/database/mysql"
+    _ "github.com/golang-migrate/migrate/source/file"
+)
+
+func main() {
+    db, _ := sql.Open("mysql", "user:password@tcp(host:port)/dbname?multiStatements=true")
+    driver, _ := mysql.WithInstance(db, &mysql.Config{})
+    m, _ := migrate.NewWithDatabaseInstance(
+        "file:///migrations",
+        "mysql", 
+        driver,
+    )
+    
+    m.Steps(2)
+}
+```
+
+## Upgrading from v1
+
+1. Write down the current migration version from schema_migrations
+1. `DROP TABLE schema_migrations`
+2. Wrap your existing migrations in transactions ([BEGIN/COMMIT](https://dev.mysql.com/doc/refman/5.7/en/commit.html)) if you use multiple statements within one migration.
+3. Download and install the latest migrate version.
+4. Force the current migration version with `migrate force <current_version>`.

database/mysql/examples/migrations/1_init.down.sql

@@ -0,0 +1 @@
+DROP TABLE IF EXISTS test;

database/mysql/examples/migrations/1_init.up.sql

@@ -0,0 +1,3 @@
+CREATE TABLE IF NOT EXISTS test (
+  firstname VARCHAR(16)
+);

database/mysql/mysql.go

@@ -1,6 +1,9 @@
+// +build go1.9
+
 package mysql
 
 import (
+	"context"
 	"crypto/tls"
 	"crypto/x509"
 	"database/sql"
@@ -10,10 +13,15 @@ import (
 	nurl "net/url"
 	"strconv"
 	"strings"
+)
 
+import (
 	"github.com/go-sql-driver/mysql"
-	"github.com/mattes/migrate"
-	"github.com/mattes/migrate/database"
+	"github.com/hashicorp/go-multierror"
+)
+
+import (
+	"github.com/golang-migrate/migrate/v4/database"
 )
 
 func init() {
@@ -23,10 +31,11 @@ func init() {
 var DefaultMigrationsTable = "schema_migrations"
 
 var (
-	ErrDatabaseDirty  = fmt.Errorf("database is dirty")
-	ErrNilConfig      = fmt.Errorf("no config")
-	ErrNoDatabaseName = fmt.Errorf("no database name")
-	ErrAppendPEM      = fmt.Errorf("failed to append PEM")
+	ErrDatabaseDirty    = fmt.Errorf("database is dirty")
+	ErrNilConfig        = fmt.Errorf("no config")
+	ErrNoDatabaseName   = fmt.Errorf("no database name")
+	ErrAppendPEM        = fmt.Errorf("failed to append PEM")
+	ErrTLSCertKeyConfig = fmt.Errorf("To use TLS client authentication, both x-tls-cert and x-tls-key must not be empty")
 )
 
 type Config struct {
@@ -35,6 +44,9 @@ type Config struct {
 }
 
 type Mysql struct {
+	// mysql RELEASE_LOCK must be called from the same conn, so
+	// just do everything over a single conn anyway.
+	conn     *sql.Conn
 	db       *sql.DB
 	isLocked bool
 
@@ -51,23 +63,31 @@ func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) {
 		return nil, err
 	}
 
-	query := `SELECT DATABASE()`
-	var databaseName sql.NullString
-	if err := instance.QueryRow(query).Scan(&databaseName); err != nil {
-		return nil, &database.Error{OrigErr: err, Query: []byte(query)}
-	}
+	if config.DatabaseName == "" {
+		query := `SELECT DATABASE()`
+		var databaseName sql.NullString
+		if err := instance.QueryRow(query).Scan(&databaseName); err != nil {
+			return nil, &database.Error{OrigErr: err, Query: []byte(query)}
+		}
 
-	if len(databaseName.String) == 0 {
-		return nil, ErrNoDatabaseName
-	}
+		if len(databaseName.String) == 0 {
+			return nil, ErrNoDatabaseName
+		}
 
-	config.DatabaseName = databaseName.String
+		config.DatabaseName = databaseName.String
+	}
 
 	if len(config.MigrationsTable) == 0 {
 		config.MigrationsTable = DefaultMigrationsTable
 	}
 
+	conn, err := instance.Conn(context.Background())
+	if err != nil {
+		return nil, err
+	}
+
 	mx := &Mysql{
+		conn:   conn,
 		db:     instance,
 		config: config,
 	}
@@ -79,33 +99,52 @@ func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) {
 	return mx, nil
 }
 
-func (m *Mysql) Open(url string) (database.Driver, error) {
-	purl, err := nurl.Parse(url)
+// extractCustomQueryParams extracts the custom query params (ones that start with "x-") from
+// mysql.Config.Params (connection parameters) as to not interfere with connecting to MySQL
+func extractCustomQueryParams(c *mysql.Config) (map[string]string, error) {
+	if c == nil {
+		return nil, ErrNilConfig
+	}
+	customQueryParams := map[string]string{}
+
+	for k, v := range c.Params {
+		if strings.HasPrefix(k, "x-") {
+			customQueryParams[k] = v
+			delete(c.Params, k)
+		}
+	}
+	return customQueryParams, nil
+}
+
+func urlToMySQLConfig(url string) (*mysql.Config, error) {
+	config, err := mysql.ParseDSN(strings.TrimPrefix(url, "mysql://"))
 	if err != nil {
 		return nil, err
 	}
 
-	q := purl.Query()
-	q.Set("multiStatements", "true")
-	purl.RawQuery = q.Encode()
+	config.MultiStatements = true
 
-	db, err := sql.Open("mysql", strings.Replace(
-		migrate.FilterCustomQuery(purl).String(), "mysql://", "", 1))
+	// Keep backwards compatibility from when we used net/url.Parse() to parse the DSN.
+	// net/url.Parse() would automatically unescape it for us.
+	// See: https://play.golang.org/p/q9j1io-YICQ
+	user, err := nurl.QueryUnescape(config.User)
 	if err != nil {
 		return nil, err
 	}
+	config.User = user
 
-	migrationsTable := purl.Query().Get("x-migrations-table")
-	if len(migrationsTable) == 0 {
-		migrationsTable = DefaultMigrationsTable
+	password, err := nurl.QueryUnescape(config.Passwd)
+	if err != nil {
+		return nil, err
 	}
+	config.Passwd = password
 
 	// use custom TLS?
-	ctls := purl.Query().Get("tls")
+	ctls := config.TLSConfig
 	if len(ctls) > 0 {
 		if _, isBool := readBool(ctls); !isBool && strings.ToLower(ctls) != "skip-verify" {
 			rootCertPool := x509.NewCertPool()
-			pem, err := ioutil.ReadFile(purl.Query().Get("x-tls-ca"))
+			pem, err := ioutil.ReadFile(config.Params["x-tls-ca"])
 			if err != nil {
 				return nil, err
 			}
@@ -114,31 +153,60 @@ func (m *Mysql) Open(url string) (database.Driver, error) {
 				return nil, ErrAppendPEM
 			}
 
-			certs, err := tls.LoadX509KeyPair(purl.Query().Get("x-tls-cert"), purl.Query().Get("x-tls-key"))
-			if err != nil {
-				return nil, err
+			clientCert := make([]tls.Certificate, 0, 1)
+			if ccert, ckey := config.Params["x-tls-cert"], config.Params["x-tls-key"]; ccert != "" || ckey != "" {
+				if ccert == "" || ckey == "" {
+					return nil, ErrTLSCertKeyConfig
+				}
+				certs, err := tls.LoadX509KeyPair(ccert, ckey)
+				if err != nil {
+					return nil, err
+				}
+				clientCert = append(clientCert, certs)
 			}
 
 			insecureSkipVerify := false
-			if len(purl.Query().Get("x-tls-insecure-skip-verify")) > 0 {
-				x, err := strconv.ParseBool(purl.Query().Get("x-tls-insecure-skip-verify"))
+			if len(config.Params["x-tls-insecure-skip-verify"]) > 0 {
+				x, err := strconv.ParseBool(config.Params["x-tls-insecure-skip-verify"])
 				if err != nil {
 					return nil, err
 				}
 				insecureSkipVerify = x
 			}
 
-			mysql.RegisterTLSConfig(ctls, &tls.Config{
+			err = mysql.RegisterTLSConfig(ctls, &tls.Config{
 				RootCAs:            rootCertPool,
-				Certificates:       []tls.Certificate{certs},
+				Certificates:       clientCert,
 				InsecureSkipVerify: insecureSkipVerify,
 			})
+			if err != nil {
+				return nil, err
+			}
 		}
 	}
 
+	return config, nil
+}
+
+func (m *Mysql) Open(url string) (database.Driver, error) {
+	config, err := urlToMySQLConfig(url)
+	if err != nil {
+		return nil, err
+	}
+
+	customParams, err := extractCustomQueryParams(config)
+	if err != nil {
+		return nil, err
+	}
+
+	db, err := sql.Open("mysql", config.FormatDSN())
+	if err != nil {
+		return nil, err
+	}
+
 	mx, err := WithInstance(db, &Config{
-		DatabaseName:    purl.Path,
-		MigrationsTable: migrationsTable,
+		DatabaseName:    config.DBName,
+		MigrationsTable: customParams["x-migrations-table"],
 	})
 	if err != nil {
 		return nil, err
@@ -148,7 +216,12 @@ func (m *Mysql) Open(url string) (database.Driver, error) {
 }
 
 func (m *Mysql) Close() error {
-	return m.db.Close()
+	connErr := m.conn.Close()
+	dbErr := m.db.Close()
+	if connErr != nil || dbErr != nil {
+		return fmt.Errorf("conn: %v, db: %v", connErr, dbErr)
+	}
+	return nil
 }
 
 func (m *Mysql) Lock() error {
@@ -156,14 +229,15 @@ func (m *Mysql) Lock() error {
 		return database.ErrLocked
 	}
 
-	aid, err := database.GenerateAdvisoryLockId(m.config.DatabaseName)
+	aid, err := database.GenerateAdvisoryLockId(
+		fmt.Sprintf("%s:%s", m.config.DatabaseName, m.config.MigrationsTable))
 	if err != nil {
 		return err
 	}
 
-	query := "SELECT GET_LOCK(?, 1)"
+	query := "SELECT GET_LOCK(?, 10)"
 	var success bool
-	if err := m.db.QueryRow(query, aid).Scan(&success); err != nil {
+	if err := m.conn.QueryRowContext(context.Background(), query, aid).Scan(&success); err != nil {
 		return &database.Error{OrigErr: err, Err: "try lock failed", Query: []byte(query)}
 	}
 
@@ -180,16 +254,21 @@ func (m *Mysql) Unlock() error {
 		return nil
 	}
 
-	aid, err := database.GenerateAdvisoryLockId(m.config.DatabaseName)
+	aid, err := database.GenerateAdvisoryLockId(
+		fmt.Sprintf("%s:%s", m.config.DatabaseName, m.config.MigrationsTable))
 	if err != nil {
 		return err
 	}
 
 	query := `SELECT RELEASE_LOCK(?)`
-	if _, err := m.db.Exec(query, aid); err != nil {
+	if _, err := m.conn.ExecContext(context.Background(), query, aid); err != nil {
 		return &database.Error{OrigErr: err, Query: []byte(query)}
 	}
 
+	// NOTE: RELEASE_LOCK could return NULL or (or 0 if the code is changed),
+	// in which case isLocked should be true until the timeout expires -- synchronizing
+	// these states is likely not worth trying to do; reconsider the necessity of isLocked.
+
 	m.isLocked = false
 	return nil
 }
@@ -201,7 +280,7 @@ func (m *Mysql) Run(migration io.Reader) error {
 	}
 
 	query := string(migr[:])
-	if _, err := m.db.Exec(query); err != nil {
+	if _, err := m.conn.ExecContext(context.Background(), query); err != nil {
 		return database.Error{OrigErr: err, Err: "migration failed", Query: migr}
 	}
 
@@ -209,20 +288,25 @@ func (m *Mysql) Run(migration io.Reader) error {
 }
 
 func (m *Mysql) SetVersion(version int, dirty bool) error {
-	tx, err := m.db.Begin()
+	tx, err := m.conn.BeginTx(context.Background(), &sql.TxOptions{})
 	if err != nil {
 		return &database.Error{OrigErr: err, Err: "transaction start failed"}
 	}
 
 	query := "TRUNCATE `" + m.config.MigrationsTable + "`"
-	if _, err := m.db.Exec(query); err != nil {
+	if _, err := tx.ExecContext(context.Background(), query); err != nil {
+		if errRollback := tx.Rollback(); errRollback != nil {
+			err = multierror.Append(err, errRollback)
+		}
 		return &database.Error{OrigErr: err, Query: []byte(query)}
 	}
 
 	if version >= 0 {
 		query := "INSERT INTO `" + m.config.MigrationsTable + "` (version, dirty) VALUES (?, ?)"
-		if _, err := m.db.Exec(query, version, dirty); err != nil {
-			tx.Rollback()
+		if _, err := tx.ExecContext(context.Background(), query, version, dirty); err != nil {
+			if errRollback := tx.Rollback(); errRollback != nil {
+				err = multierror.Append(err, errRollback)
+			}
 			return &database.Error{OrigErr: err, Query: []byte(query)}
 		}
 	}
@@ -236,7 +320,7 @@ func (m *Mysql) SetVersion(version int, dirty bool) error {
 
 func (m *Mysql) Version() (version int, dirty bool, err error) {
 	query := "SELECT version, dirty FROM `" + m.config.MigrationsTable + "` LIMIT 1"
-	err = m.db.QueryRow(query).Scan(&version, &dirty)
+	err = m.conn.QueryRowContext(context.Background(), query).Scan(&version, &dirty)
 	switch {
 	case err == sql.ErrNoRows:
 		return database.NilVersion, false, nil
@@ -254,14 +338,18 @@ func (m *Mysql) Version() (version int, dirty bool, err error) {
 	}
 }
 
-func (m *Mysql) Drop() error {
+func (m *Mysql) Drop() (err error) {
 	// select all tables
 	query := `SHOW TABLES LIKE '%'`
-	tables, err := m.db.Query(query)
+	tables, err := m.conn.QueryContext(context.Background(), query)
 	if err != nil {
 		return &database.Error{OrigErr: err, Query: []byte(query)}
 	}
-	defer tables.Close()
+	defer func() {
+		if errClose := tables.Close(); errClose != nil {
+			err = multierror.Append(err, errClose)
+		}
+	}()
 
 	// delete one table after another
 	tableNames := make([]string, 0)
@@ -276,26 +364,51 @@ func (m *Mysql) Drop() error {
 	}
 
 	if len(tableNames) > 0 {
+		// disable checking foreign key constraints until finished
+		query = `SET foreign_key_checks = 0`
+		if _, err := m.conn.ExecContext(context.Background(), query); err != nil {
+			return &database.Error{OrigErr: err, Query: []byte(query)}
+		}
+
+		defer func() {
+			// enable foreign key checks
+			_, _ = m.conn.ExecContext(context.Background(), `SET foreign_key_checks = 1`)
+		}()
+
 		// delete one by one ...
 		for _, t := range tableNames {
-			query = "DROP TABLE IF EXISTS `" + t + "` CASCADE"
-			if _, err := m.db.Exec(query); err != nil {
+			query = "DROP TABLE IF EXISTS `" + t + "`"
+			if _, err := m.conn.ExecContext(context.Background(), query); err != nil {
 				return &database.Error{OrigErr: err, Query: []byte(query)}
 			}
 		}
-		if err := m.ensureVersionTable(); err != nil {
-			return err
-		}
 	}
 
 	return nil
 }
 
-func (m *Mysql) ensureVersionTable() error {
+// ensureVersionTable checks if versions table exists and, if not, creates it.
+// Note that this function locks the database, which deviates from the usual
+// convention of "caller locks" in the Mysql type.
+func (m *Mysql) ensureVersionTable() (err error) {
+	if err = m.Lock(); err != nil {
+		return err
+	}
+
+	defer func() {
+		if e := m.Unlock(); e != nil {
+			if err == nil {
+				err = e
+			} else {
+				err = multierror.Append(err, e)
+			}
+		}
+	}()
+
 	// check if migration table exists
 	var result string
 	query := `SHOW TABLES LIKE "` + m.config.MigrationsTable + `"`
-	if err := m.db.QueryRow(query).Scan(&result); err != nil {
+	if err := m.conn.QueryRowContext(context.Background(), query).Scan(&result); err != nil {
 		if err != sql.ErrNoRows {
 			return &database.Error{OrigErr: err, Query: []byte(query)}
 		}
@@ -305,7 +418,7 @@ func (m *Mysql) ensureVersionTable() error {
 
 	// if not, create the empty migration table
 	query = "CREATE TABLE `" + m.config.MigrationsTable + "` (version bigint not null primary key, dirty boolean not null)"
-	if _, err := m.db.Exec(query); err != nil {
+	if _, err := m.conn.ExecContext(context.Background(), query); err != nil {
 		return &database.Error{OrigErr: err, Query: []byte(query)}
 	}
 	return nil

database/mysql/mysql_test.go

@@ -1,34 +1,65 @@
 package mysql
 
 import (
+	"context"
 	"database/sql"
 	sqldriver "database/sql/driver"
 	"fmt"
-	// "io/ioutil"
-	// "log"
+	"log"
 	"testing"
-
-	// "github.com/go-sql-driver/mysql"
-	dt "github.com/mattes/migrate/database/testing"
-	mt "github.com/mattes/migrate/testing"
 )
 
-var versions = []mt.Version{
-	{"mysql:8", []string{"MYSQL_ROOT_PASSWORD=root", "MYSQL_DATABASE=public"}},
-	{"mysql:5.7", []string{"MYSQL_ROOT_PASSWORD=root", "MYSQL_DATABASE=public"}},
-	{"mysql:5.6", []string{"MYSQL_ROOT_PASSWORD=root", "MYSQL_DATABASE=public"}},
-	{"mysql:5.5", []string{"MYSQL_ROOT_PASSWORD=root", "MYSQL_DATABASE=public"}},
-}
+import (
+	"github.com/dhui/dktest"
+	"github.com/go-sql-driver/mysql"
+	"github.com/stretchr/testify/assert"
+)
 
-func isReady(i mt.Instance) bool {
-	db, err := sql.Open("mysql", fmt.Sprintf("root:root@tcp(%v:%v)/public", i.Host(), i.Port()))
+import (
+	"github.com/golang-migrate/migrate/v4"
+	dt "github.com/golang-migrate/migrate/v4/database/testing"
+	"github.com/golang-migrate/migrate/v4/dktesting"
+	_ "github.com/golang-migrate/migrate/v4/source/file"
+)
+
+const defaultPort = 3306
+
+var (
+	opts = dktest.Options{
+		Env:          map[string]string{"MYSQL_ROOT_PASSWORD": "root", "MYSQL_DATABASE": "public"},
+		PortRequired: true, ReadyFunc: isReady,
+	}
+	// Supported versions: https://www.mysql.com/support/supportedplatforms/database.html
+	specs = []dktesting.ContainerSpec{
+		{ImageName: "mysql:5.5", Options: opts},
+		{ImageName: "mysql:5.6", Options: opts},
+		{ImageName: "mysql:5.7", Options: opts},
+		{ImageName: "mysql:8", Options: opts},
+	}
+)
+
+func isReady(ctx context.Context, c dktest.ContainerInfo) bool {
+	ip, port, err := c.Port(defaultPort)
 	if err != nil {
 		return false
 	}
-	defer db.Close()
-	err = db.Ping()
 
-	if err == sqldriver.ErrBadConn {
+	db, err := sql.Open("mysql", fmt.Sprintf("root:root@tcp(%v:%v)/public", ip, port))
+	if err != nil {
+		return false
+	}
+	defer func() {
+		if err := db.Close(); err != nil {
+			log.Println("close error:", err)
+		}
+	}()
+	if err = db.PingContext(ctx); err != nil {
+		switch err {
+		case sqldriver.ErrBadConn, mysql.ErrInvalidConn:
+			return false
+		default:
+			fmt.Println(err)
+		}
 		return false
 	}
 
@@ -38,23 +69,210 @@ func isReady(i mt.Instance) bool {
 func Test(t *testing.T) {
 	// mysql.SetLogger(mysql.Logger(log.New(ioutil.Discard, "", log.Ltime)))
 
-	mt.ParallelTest(t, versions, isReady,
-		func(t *testing.T, i mt.Instance) {
-			p := &Mysql{}
-			addr := fmt.Sprintf("mysql://root:root@tcp(%v:%v)/public", i.Host(), i.Port())
-			d, err := p.Open(addr)
-			if err != nil {
-				t.Fatalf("%v", err)
-			}
-			dt.Test(t, d, []byte("SELECT 1"))
+	dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
+		ip, port, err := c.Port(defaultPort)
+		if err != nil {
+			t.Fatal(err)
+		}
 
-			// check ensureVersionTable
-			if err := d.(*Mysql).ensureVersionTable(); err != nil {
-				t.Fatal(err)
+		addr := fmt.Sprintf("mysql://root:root@tcp(%v:%v)/public", ip, port)
+		p := &Mysql{}
+		d, err := p.Open(addr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer func() {
+			if err := d.Close(); err != nil {
+				t.Error(err)
 			}
-			// check again
-			if err := d.(*Mysql).ensureVersionTable(); err != nil {
-				t.Fatal(err)
+		}()
+		dt.Test(t, d, []byte("SELECT 1"))
+
+		// check ensureVersionTable
+		if err := d.(*Mysql).ensureVersionTable(); err != nil {
+			t.Fatal(err)
+		}
+		// check again
+		if err := d.(*Mysql).ensureVersionTable(); err != nil {
+			t.Fatal(err)
+		}
+	})
+}
+
+func TestMigrate(t *testing.T) {
+	// mysql.SetLogger(mysql.Logger(log.New(ioutil.Discard, "", log.Ltime)))
+
+	dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
+		ip, port, err := c.Port(defaultPort)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		addr := fmt.Sprintf("mysql://root:root@tcp(%v:%v)/public", ip, port)
+		p := &Mysql{}
+		d, err := p.Open(addr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer func() {
+			if err := d.Close(); err != nil {
+				t.Error(err)
+			}
+		}()
+
+		m, err := migrate.NewWithDatabaseInstance("file://./examples/migrations", "public", d)
+		if err != nil {
+			t.Fatal(err)
+		}
+		dt.TestMigrate(t, m, []byte("SELECT 1"))
+
+		// check ensureVersionTable
+		if err := d.(*Mysql).ensureVersionTable(); err != nil {
+			t.Fatal(err)
+		}
+		// check again
+		if err := d.(*Mysql).ensureVersionTable(); err != nil {
+			t.Fatal(err)
+		}
+	})
+}
+
+func TestLockWorks(t *testing.T) {
+	dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
+		ip, port, err := c.Port(defaultPort)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		addr := fmt.Sprintf("mysql://root:root@tcp(%v:%v)/public", ip, port)
+		p := &Mysql{}
+		d, err := p.Open(addr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		dt.Test(t, d, []byte("SELECT 1"))
+
+		ms := d.(*Mysql)
+
+		err = ms.Lock()
+		if err != nil {
+			t.Fatal(err)
+		}
+		err = ms.Unlock()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// make sure the 2nd lock works (RELEASE_LOCK is very finicky)
+		err = ms.Lock()
+		if err != nil {
+			t.Fatal(err)
+		}
+		err = ms.Unlock()
+		if err != nil {
+			t.Fatal(err)
+		}
+	})
+}
+
+func TestExtractCustomQueryParams(t *testing.T) {
+	testcases := []struct {
+		name                 string
+		config               *mysql.Config
+		expectedParams       map[string]string
+		expectedCustomParams map[string]string
+		expectedErr          error
+	}{
+		{name: "nil config", expectedErr: ErrNilConfig},
+		{
+			name:                 "no params",
+			config:               mysql.NewConfig(),
+			expectedCustomParams: map[string]string{},
+		},
+		{
+			name:                 "no custom params",
+			config:               &mysql.Config{Params: map[string]string{"hello": "world"}},
+			expectedParams:       map[string]string{"hello": "world"},
+			expectedCustomParams: map[string]string{},
+		},
+		{
+			name: "one param, one custom param",
+			config: &mysql.Config{
+				Params: map[string]string{"hello": "world", "x-foo": "bar"},
+			},
+			expectedParams:       map[string]string{"hello": "world"},
+			expectedCustomParams: map[string]string{"x-foo": "bar"},
+		},
+		{
+			name: "multiple params, multiple custom params",
+			config: &mysql.Config{
+				Params: map[string]string{
+					"hello": "world",
+					"x-foo": "bar",
+					"dead":  "beef",
+					"x-cat": "hat",
+				},
+			},
+			expectedParams:       map[string]string{"hello": "world", "dead": "beef"},
+			expectedCustomParams: map[string]string{"x-foo": "bar", "x-cat": "hat"},
+		},
+	}
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			customParams, err := extractCustomQueryParams(tc.config)
+			if tc.config != nil {
+				assert.Equal(t, tc.expectedParams, tc.config.Params,
+					"Expected config params have custom params properly removed")
+			}
+			assert.Equal(t, tc.expectedErr, err, "Expected errors to match")
+			assert.Equal(t, tc.expectedCustomParams, customParams,
+				"Expected custom params to be properly extracted")
+		})
+	}
+}
+
+func TestURLToMySQLConfig(t *testing.T) {
+	testcases := []struct {
+		name        string
+		urlStr      string
+		expectedDSN string // empty string signifies that an error is expected
+	}{
+		{name: "no user/password", urlStr: "mysql://tcp(127.0.0.1:3306)/myDB?multiStatements=true",
+			expectedDSN: "tcp(127.0.0.1:3306)/myDB?multiStatements=true"},
+		{name: "only user", urlStr: "mysql://username@tcp(127.0.0.1:3306)/myDB?multiStatements=true",
+			expectedDSN: "username@tcp(127.0.0.1:3306)/myDB?multiStatements=true"},
+		{name: "only user - with encoded :",
+			urlStr:      "mysql://username%3A@tcp(127.0.0.1:3306)/myDB?multiStatements=true",
+			expectedDSN: "username:@tcp(127.0.0.1:3306)/myDB?multiStatements=true"},
+		{name: "only user - with encoded @",
+			urlStr:      "mysql://username%40@tcp(127.0.0.1:3306)/myDB?multiStatements=true",
+			expectedDSN: "username@@tcp(127.0.0.1:3306)/myDB?multiStatements=true"},
+		{name: "user/password", urlStr: "mysql://username:password@tcp(127.0.0.1:3306)/myDB?multiStatements=true",
+			expectedDSN: "username:password@tcp(127.0.0.1:3306)/myDB?multiStatements=true"},
+		// Not supported yet: https://github.com/go-sql-driver/mysql/issues/591
+		// {name: "user/password - user with encoded :",
+		// 	urlStr:      "mysql://username%3A:password@tcp(127.0.0.1:3306)/myDB?multiStatements=true",
+		// 	expectedDSN: "username::pasword@tcp(127.0.0.1:3306)/myDB?multiStatements=true"},
+		{name: "user/password - user with encoded @",
+			urlStr:      "mysql://username%40:password@tcp(127.0.0.1:3306)/myDB?multiStatements=true",
+			expectedDSN: "username@:password@tcp(127.0.0.1:3306)/myDB?multiStatements=true"},
+		{name: "user/password - password with encoded :",
+			urlStr:      "mysql://username:password%3A@tcp(127.0.0.1:3306)/myDB?multiStatements=true",
+			expectedDSN: "username:password:@tcp(127.0.0.1:3306)/myDB?multiStatements=true"},
+		{name: "user/password - password with encoded @",
+			urlStr:      "mysql://username:password%40@tcp(127.0.0.1:3306)/myDB?multiStatements=true",
+			expectedDSN: "username:password@@tcp(127.0.0.1:3306)/myDB?multiStatements=true"},
+	}
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			config, err := urlToMySQLConfig(tc.urlStr)
+			if err != nil {
+				t.Fatal("Failed to parse url string:", tc.urlStr, "error:", err)
+			}
+			dsn := config.FormatDSN()
+			if dsn != tc.expectedDSN {
+				t.Error("Got unexpected DSN:", dsn, "!=", tc.expectedDSN)
 			}
 		})
+	}
 }

database/parse_test.go

@@ -0,0 +1,241 @@
+package database_test
+
+import (
+	"encoding/hex"
+	"net/url"
+	"strings"
+	"testing"
+)
+
+const reservedChars = "!#$%&'()*+,/:;=?@[]"
+
+const baseUsername = "username"
+
+// TestUserUnencodedReservedURLChars documents the behavior of using unencoded reserved characters in usernames with
+// net/url Parse()
+func TestUserUnencodedReservedURLChars(t *testing.T) {
+	scheme := "database://"
+	urlSuffix := "password@localhost:12345/myDB?someParam=true"
+	urlSuffixAndSep := ":" + urlSuffix
+
+	testcases := []struct {
+		char             string
+		parses           bool
+		expectedUsername string // empty string means that the username failed to parse
+		encodedURL       string
+	}{
+		{char: "!", parses: true, expectedUsername: baseUsername + "!",
+			encodedURL: scheme + baseUsername + "%21" + urlSuffixAndSep},
+		{char: "#", parses: true, expectedUsername: "",
+			encodedURL: scheme + baseUsername + "#" + urlSuffixAndSep},
+		{char: "$", parses: true, expectedUsername: baseUsername + "$",
+			encodedURL: scheme + baseUsername + "$" + urlSuffixAndSep},
+		{char: "%", parses: false},
+		{char: "&", parses: true, expectedUsername: baseUsername + "&",
+			encodedURL: scheme + baseUsername + "&" + urlSuffixAndSep},
+		{char: "'", parses: true, expectedUsername: "username'",
+			encodedURL: scheme + baseUsername + "%27" + urlSuffixAndSep},
+		{char: "(", parses: true, expectedUsername: "username(",
+			encodedURL: scheme + baseUsername + "%28" + urlSuffixAndSep},
+		{char: ")", parses: true, expectedUsername: "username)",
+			encodedURL: scheme + baseUsername + "%29" + urlSuffixAndSep},
+		{char: "*", parses: true, expectedUsername: "username*",
+			encodedURL: scheme + baseUsername + "%2A" + urlSuffixAndSep},
+		{char: "+", parses: true, expectedUsername: "username+",
+			encodedURL: scheme + baseUsername + "+" + urlSuffixAndSep},
+		{char: ",", parses: true, expectedUsername: "username,",
+			encodedURL: scheme + baseUsername + "," + urlSuffixAndSep},
+		{char: "/", parses: true, expectedUsername: "",
+			encodedURL: scheme + baseUsername + "/" + urlSuffixAndSep},
+		{char: ":", parses: true, expectedUsername: baseUsername,
+			encodedURL: scheme + baseUsername + ":%3A" + urlSuffix},
+		{char: ";", parses: true, expectedUsername: "username;",
+			encodedURL: scheme + baseUsername + ";" + urlSuffixAndSep},
+		{char: "=", parses: true, expectedUsername: "username=",
+			encodedURL: scheme + baseUsername + "=" + urlSuffixAndSep},
+		{char: "?", parses: true, expectedUsername: "",
+			encodedURL: scheme + baseUsername + "?" + urlSuffixAndSep},
+		{char: "@", parses: true, expectedUsername: "username@",
+			encodedURL: scheme + baseUsername + "%40" + urlSuffixAndSep},
+		{char: "[", parses: false},
+		{char: "]", parses: false},
+	}
+
+	testedChars := make([]string, 0, len(reservedChars))
+	for _, tc := range testcases {
+		testedChars = append(testedChars, tc.char)
+		t.Run("reserved char "+tc.char, func(t *testing.T) {
+			s := scheme + baseUsername + tc.char + urlSuffixAndSep
+			u, err := url.Parse(s)
+			if err == nil {
+				if !tc.parses {
+					t.Error("Unexpectedly parsed reserved character. url:", s)
+					return
+				}
+				var username string
+				if u.User != nil {
+					username = u.User.Username()
+				}
+				if username != tc.expectedUsername {
+					t.Error("Got unexpected username:", username, "!=", tc.expectedUsername)
+				}
+				if s := u.String(); s != tc.encodedURL {
+					t.Error("Got unexpected encoded URL:", s, "!=", tc.encodedURL)
+				}
+			} else {
+				if tc.parses {
+					t.Error("Failed to parse reserved character. url:", s)
+				}
+			}
+		})
+	}
+
+	t.Run("All reserved chars tested", func(t *testing.T) {
+		if s := strings.Join(testedChars, ""); s != reservedChars {
+			t.Error("Not all reserved URL characters were tested:", s, "!=", reservedChars)
+		}
+	})
+}
+
+func TestUserEncodedReservedURLChars(t *testing.T) {
+	scheme := "database://"
+	urlSuffix := "password@localhost:12345/myDB?someParam=true"
+	urlSuffixAndSep := ":" + urlSuffix
+
+	for _, c := range reservedChars {
+		c := string(c)
+		t.Run("reserved char "+c, func(t *testing.T) {
+			encodedChar := "%" + hex.EncodeToString([]byte(c))
+			s := scheme + baseUsername + encodedChar + urlSuffixAndSep
+			expectedUsername := baseUsername + c
+			u, err := url.Parse(s)
+			if err != nil {
+				t.Fatal("Failed to parse url with encoded reserved character. url:", s)
+			}
+			if u.User == nil {
+				t.Fatal("Failed to parse userinfo with encoded reserve character. url:", s)
+			}
+			if username := u.User.Username(); username != expectedUsername {
+				t.Fatal("Got unexpected username:", username, "!=", expectedUsername)
+			}
+		})
+	}
+}
+
+// TestPasswordUnencodedReservedURLChars documents the behavior of using unencoded reserved characters in passwords
+// with net/url Parse()
+func TestPasswordUnencodedReservedURLChars(t *testing.T) {
+	username := baseUsername
+	schemeAndUsernameAndSep := "database://" + username + ":"
+	basePassword := "password"
+	urlSuffixAndSep := "@localhost:12345/myDB?someParam=true"
+
+	testcases := []struct {
+		char             string
+		parses           bool
+		expectedUsername string // empty string means that the username failed to parse
+		expectedPassword string // empty string means that the password failed to parse
+		encodedURL       string
+	}{
+		{char: "!", parses: true, expectedUsername: username, expectedPassword: basePassword + "!",
+			encodedURL: schemeAndUsernameAndSep + basePassword + "%21" + urlSuffixAndSep},
+		{char: "#", parses: false},
+		{char: "$", parses: true, expectedUsername: username, expectedPassword: basePassword + "$",
+			encodedURL: schemeAndUsernameAndSep + basePassword + "$" + urlSuffixAndSep},
+		{char: "%", parses: false},
+		{char: "&", parses: true, expectedUsername: username, expectedPassword: basePassword + "&",
+			encodedURL: schemeAndUsernameAndSep + basePassword + "&" + urlSuffixAndSep},
+		{char: "'", parses: true, expectedUsername: username, expectedPassword: "password'",
+			encodedURL: schemeAndUsernameAndSep + basePassword + "%27" + urlSuffixAndSep},
+		{char: "(", parses: true, expectedUsername: username, expectedPassword: "password(",
+			encodedURL: schemeAndUsernameAndSep + basePassword + "%28" + urlSuffixAndSep},
+		{char: ")", parses: true, expectedUsername: username, expectedPassword: "password)",
+			encodedURL: schemeAndUsernameAndSep + basePassword + "%29" + urlSuffixAndSep},
+		{char: "*", parses: true, expectedUsername: username, expectedPassword: "password*",
+			encodedURL: schemeAndUsernameAndSep + basePassword + "%2A" + urlSuffixAndSep},
+		{char: "+", parses: true, expectedUsername: username, expectedPassword: "password+",
+			encodedURL: schemeAndUsernameAndSep + basePassword + "+" + urlSuffixAndSep},
+		{char: ",", parses: true, expectedUsername: username, expectedPassword: "password,",
+			encodedURL: schemeAndUsernameAndSep + basePassword + "," + urlSuffixAndSep},
+		{char: "/", parses: false},
+		{char: ":", parses: true, expectedUsername: username, expectedPassword: "password:",
+			encodedURL: schemeAndUsernameAndSep + basePassword + "%3A" + urlSuffixAndSep},
+		{char: ";", parses: true, expectedUsername: username, expectedPassword: "password;",
+			encodedURL: schemeAndUsernameAndSep + basePassword + ";" + urlSuffixAndSep},
+		{char: "=", parses: true, expectedUsername: username, expectedPassword: "password=",
+			encodedURL: schemeAndUsernameAndSep + basePassword + "=" + urlSuffixAndSep},
+		{char: "?", parses: false},
+		{char: "@", parses: true, expectedUsername: username, expectedPassword: "password@",
+			encodedURL: schemeAndUsernameAndSep + basePassword + "%40" + urlSuffixAndSep},
+		{char: "[", parses: false},
+		{char: "]", parses: false},
+	}
+
+	testedChars := make([]string, 0, len(reservedChars))
+	for _, tc := range testcases {
+		testedChars = append(testedChars, tc.char)
+		t.Run("reserved char "+tc.char, func(t *testing.T) {
+			s := schemeAndUsernameAndSep + basePassword + tc.char + urlSuffixAndSep
+			u, err := url.Parse(s)
+			if err == nil {
+				if !tc.parses {
+					t.Error("Unexpectedly parsed reserved character. url:", s)
+					return
+				}
+				var username, password string
+				if u.User != nil {
+					username = u.User.Username()
+					password, _ = u.User.Password()
+				}
+				if username != tc.expectedUsername {
+					t.Error("Got unexpected username:", username, "!=", tc.expectedUsername)
+				}
+				if password != tc.expectedPassword {
+					t.Error("Got unexpected password:", password, "!=", tc.expectedPassword)
+				}
+				if s := u.String(); s != tc.encodedURL {
+					t.Error("Got unexpected encoded URL:", s, "!=", tc.encodedURL)
+				}
+			} else {
+				if tc.parses {
+					t.Error("Failed to parse reserved character. url:", s)
+				}
+			}
+		})
+	}
+
+	t.Run("All reserved chars tested", func(t *testing.T) {
+		if s := strings.Join(testedChars, ""); s != reservedChars {
+			t.Error("Not all reserved URL characters were tested:", s, "!=", reservedChars)
+		}
+	})
+}
+
+func TestPasswordEncodedReservedURLChars(t *testing.T) {
+	username := baseUsername
+	schemeAndUsernameAndSep := "database://" + username + ":"
+	basePassword := "password"
+	urlSuffixAndSep := "@localhost:12345/myDB?someParam=true"
+
+	for _, c := range reservedChars {
+		c := string(c)
+		t.Run("reserved char "+c, func(t *testing.T) {
+			encodedChar := "%" + hex.EncodeToString([]byte(c))
+			s := schemeAndUsernameAndSep + basePassword + encodedChar + urlSuffixAndSep
+			expectedPassword := basePassword + c
+			u, err := url.Parse(s)
+			if err != nil {
+				t.Fatal("Failed to parse url with encoded reserved character. url:", s)
+			}
+			if u.User == nil {
+				t.Fatal("Failed to parse userinfo with encoded reserve character. url:", s)
+			}
+			if n := u.User.Username(); n != username {
+				t.Fatal("Got unexpected username:", n, "!=", username)
+			}
+			if p, _ := u.User.Password(); p != expectedPassword {
+				t.Fatal("Got unexpected password:", p, "!=", expectedPassword)
+			}
+		})
+	}
+}

database/postgres/TUTORIAL.md

@@ -0,0 +1,148 @@
+# PostgreSQL tutorial for beginners
+
+## Create/configure database
+
+For the purpose of this tutorial let's create PostgreSQL database called `example`.
+Our user here is `postgres`, password `password`, and host is `localhost`.
+```
+psql -h localhost -U postgres -w -c "create database example;"
+```
+When using Migrate CLI we need to pass to database URL. Let's export it to a variable for convienience:
+```
+export POSTGRESQL_URL=postgres://postgres:password@localhost:5432/example?sslmode=disable
+```
+`sslmode=disable` means that the connection with our database will not be encrypted. Enabling it is left as an exercise.
+
+You can find further description of database URLs [here](README.md#database-urls).
+
+## Create migrations
+Let's create table called `users`:
+```
+migrate create -ext sql -dir db/migrations -seq create_users_table
+```
+If there were no errors, we should have two files available under `db/migrations` folder:
+- 000001_create_users_table.down.sql
+- 000001_create_users_table.up.sql
+
+Note the `sql` extension that we provided.
+
+In the `.up.sql` file let's create the table:
+```
+CREATE TABLE IF NOT EXISTS users(
+   user_id serial PRIMARY KEY,
+   username VARCHAR (50) UNIQUE NOT NULL,
+   password VARCHAR (50) NOT NULL,
+   email VARCHAR (300) UNIQUE NOT NULL
+);
+```
+And in the `.down.sql` let's delete it:
+```
+DROP TABLE IF EXISTS users;
+```
+By adding `IF EXISTS/IF NOT EXISTS` we are making migrations idempotent - you can read more about idempotency in [getting started](GETTING_STARTED.md#create-migrations)
+
+## Run migrations
+```
+migrate -database ${POSTGRESQL_URL} -path db/migrations up
+```
+Let's check if the table was created properly by running `psql example -c "\d users"`.
+The output you are supposed to see:
+```
+                                    Table "public.users"
+  Column  |          Type          |                        Modifiers                        
+----------+------------------------+---------------------------------------------------------
+ user_id  | integer                | not null default nextval('users_user_id_seq'::regclass)
+ username | character varying(50)  | not null
+ password | character varying(50)  | not null
+ email    | character varying(300) | not null
+Indexes:
+    "users_pkey" PRIMARY KEY, btree (user_id)
+    "users_email_key" UNIQUE CONSTRAINT, btree (email)
+    "users_username_key" UNIQUE CONSTRAINT, btree (username)
+```
+Great! Now let's check if running reverse migration also works:
+```
+migrate -database ${POSTGRESQL_URL} -path db/migrations down
+```
+Make sure to check if your database changed as expected in this case as well.
+
+## Database transactions
+
+To show database transactions usage, let's create another set of migrations by running:
+```
+migrate create -ext sql -dir db/migrations -seq add_mood_to_users
+```
+Again, it should create for us two migrations files:
+- 000002_add_mood_to_users.down.sql
+- 000002_add_mood_to_users.up.sql
+
+In Postgres, when we want our queries to be done in a transaction, we need to wrap it with `BEGIN` and `COMMIT` commands.
+In our example, we are going to add a column to our database that can only accept enumerable values or NULL.
+Migration up:
+```
+BEGIN;
+
+CREATE TYPE enum_mood AS ENUM (
+	'happy',
+	'sad',
+	'neutral'
+);
+ALTER TABLE users ADD COLUMN mood enum_mood;
+
+COMMIT;
+```
+Migration down:
+```
+BEGIN;
+
+ALTER TABLE users DROP COLUMN mood;
+DROP TYPE enum_mood;
+
+COMMIT;
+```
+
+Now we can run our new migration and check the database:
+```
+migrate -database ${POSTGRESQL_URL} -path db/migrations up
+psql example -c "\d users"
+```
+Expected output:
+```
+                                    Table "public.users"
+  Column  |          Type          |                        Modifiers                        
+----------+------------------------+---------------------------------------------------------
+ user_id  | integer                | not null default nextval('users_user_id_seq'::regclass)
+ username | character varying(50)  | not null
+ password | character varying(50)  | not null
+ email    | character varying(300) | not null
+ mood     | enum_mood              | 
+Indexes:
+    "users_pkey" PRIMARY KEY, btree (user_id)
+    "users_email_key" UNIQUE CONSTRAINT, btree (email)
+    "users_username_key" UNIQUE CONSTRAINT, btree (username)
+```
+
+## Optional: Run migrations within your Go app
+Here is a very simple app running migrations for the above configuration:
+```
+import (
+	"log"
+
+	"github.com/golang-migrate/migrate/v4"
+	_ "github.com/golang-migrate/migrate/v4/database/postgres"
+	_ "github.com/golang-migrate/migrate/v4/source/file"
+)
+
+func main() {
+	m, err := migrate.New(
+		"file://db/migrations",
+		"postgres://postgres:postgres@localhost:5432/example?sslmode=disable")
+	if err != nil {
+		log.Fatal(err)
+	}
+	if err := m.Up(); err != nil {
+		log.Fatal(err)
+	}
+}
+```
+You can find details [here](README.md#use-in-your-go-project)

database/postgres/postgres.go

@@ -1,15 +1,21 @@
+// +build go1.9
+
 package postgres
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
 	"io"
 	"io/ioutil"
 	nurl "net/url"
+	"strconv"
+	"strings"
 
+	"github.com/golang-migrate/migrate/v4"
+	"github.com/golang-migrate/migrate/v4/database"
+	multierror "github.com/hashicorp/go-multierror"
 	"github.com/lib/pq"
-	"github.com/mattes/migrate"
-	"github.com/mattes/migrate/database"
 )
 
 func init() {
@@ -30,13 +36,16 @@ var (
 type Config struct {
 	MigrationsTable string
 	DatabaseName    string
+	SchemaName      string
 }
 
 type Postgres struct {
+	// Locking and unlocking need to use the same connection
+	conn     *sql.Conn
 	db       *sql.DB
 	isLocked bool
 
-	// Open and WithInstance need to garantuee that config is never nil
+	// Open and WithInstance need to guarantee that config is never nil
 	config *Config
 }
 
@@ -49,23 +58,46 @@ func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) {
 		return nil, err
 	}
 
-	query := `SELECT CURRENT_DATABASE()`
-	var databaseName string
-	if err := instance.QueryRow(query).Scan(&databaseName); err != nil {
-		return nil, &database.Error{OrigErr: err, Query: []byte(query)}
+	if config.DatabaseName == "" {
+		query := `SELECT CURRENT_DATABASE()`
+		var databaseName string
+		if err := instance.QueryRow(query).Scan(&databaseName); err != nil {
+			return nil, &database.Error{OrigErr: err, Query: []byte(query)}
+		}
+
+		if len(databaseName) == 0 {
+			return nil, ErrNoDatabaseName
+		}
+
+		config.DatabaseName = databaseName
 	}
 
-	if len(databaseName) == 0 {
-		return nil, ErrNoDatabaseName
-	}
+	if config.SchemaName == "" {
+		query := `SELECT CURRENT_SCHEMA()`
+		var schemaName string
+		if err := instance.QueryRow(query).Scan(&schemaName); err != nil {
+			return nil, &database.Error{OrigErr: err, Query: []byte(query)}
+		}
 
-	config.DatabaseName = databaseName
+		if len(schemaName) == 0 {
+			return nil, ErrNoSchema
+		}
+
+		config.SchemaName = schemaName
+	}
 
 	if len(config.MigrationsTable) == 0 {
 		config.MigrationsTable = DefaultMigrationsTable
 	}
 
+	conn, err := instance.Conn(context.Background())
+
+	if err != nil {
+		return nil, err
+	}
+
 	px := &Postgres{
+		conn:   conn,
 		db:     instance,
 		config: config,
 	}
@@ -89,14 +121,12 @@ func (p *Postgres) Open(url string) (database.Driver, error) {
 	}
 
 	migrationsTable := purl.Query().Get("x-migrations-table")
-	if len(migrationsTable) == 0 {
-		migrationsTable = DefaultMigrationsTable
-	}
 
 	px, err := WithInstance(db, &Config{
 		DatabaseName:    purl.Path,
 		MigrationsTable: migrationsTable,
 	})
+
 	if err != nil {
 		return nil, err
 	}
@@ -105,7 +135,12 @@ func (p *Postgres) Open(url string) (database.Driver, error) {
 }
 
 func (p *Postgres) Close() error {
-	return p.db.Close()
+	connErr := p.conn.Close()
+	dbErr := p.db.Close()
+	if connErr != nil || dbErr != nil {
+		return fmt.Errorf("conn: %v, db: %v", connErr, dbErr)
+	}
+	return nil
 }
 
 // https://www.postgresql.org/docs/9.6/static/explicit-locking.html#ADVISORY-LOCKS
@@ -114,25 +149,19 @@ func (p *Postgres) Lock() error {
 		return database.ErrLocked
 	}
 
-	aid, err := database.GenerateAdvisoryLockId(p.config.DatabaseName)
+	aid, err := database.GenerateAdvisoryLockId(p.config.DatabaseName, p.config.SchemaName)
 	if err != nil {
 		return err
 	}
 
-	// This will either obtain the lock immediately and return true,
-	// or return false if the lock cannot be acquired immediately.
-	query := `SELECT pg_try_advisory_lock($1)`
-	var success bool
-	if err := p.db.QueryRow(query, aid).Scan(&success); err != nil {
+	// This will wait indefinitely until the lock can be acquired.
+	query := `SELECT pg_advisory_lock($1)`
+	if _, err := p.conn.ExecContext(context.Background(), query, aid); err != nil {
 		return &database.Error{OrigErr: err, Err: "try lock failed", Query: []byte(query)}
 	}
 
-	if success {
-		p.isLocked = true
-		return nil
-	}
-
-	return database.ErrLocked
+	p.isLocked = true
+	return nil
 }
 
 func (p *Postgres) Unlock() error {
@@ -140,13 +169,13 @@ func (p *Postgres) Unlock() error {
 		return nil
 	}
 
-	aid, err := database.GenerateAdvisoryLockId(p.config.DatabaseName)
+	aid, err := database.GenerateAdvisoryLockId(p.config.DatabaseName, p.config.SchemaName)
 	if err != nil {
 		return err
 	}
 
 	query := `SELECT pg_advisory_unlock($1)`
-	if _, err := p.db.Exec(query, aid); err != nil {
+	if _, err := p.conn.ExecContext(context.Background(), query, aid); err != nil {
 		return &database.Error{OrigErr: err, Query: []byte(query)}
 	}
 	p.isLocked = false
@@ -161,30 +190,86 @@ func (p *Postgres) Run(migration io.Reader) error {
 
 	// run migration
 	query := string(migr[:])
-	if _, err := p.db.Exec(query); err != nil {
-		// TODO: cast to postgress error and get line number
+	if _, err := p.conn.ExecContext(context.Background(), query); err != nil {
+		if pgErr, ok := err.(*pq.Error); ok {
+			var line uint
+			var col uint
+			var lineColOK bool
+			if pgErr.Position != "" {
+				if pos, err := strconv.ParseUint(pgErr.Position, 10, 64); err == nil {
+					line, col, lineColOK = computeLineFromPos(query, int(pos))
+				}
+			}
+			message := fmt.Sprintf("migration failed: %s", pgErr.Message)
+			if lineColOK {
+				message = fmt.Sprintf("%s (column %d)", message, col)
+			}
+			if pgErr.Detail != "" {
+				message = fmt.Sprintf("%s, %s", message, pgErr.Detail)
+			}
+			return database.Error{OrigErr: err, Err: message, Query: migr, Line: line}
+		}
 		return database.Error{OrigErr: err, Err: "migration failed", Query: migr}
 	}
 
 	return nil
 }
 
+func computeLineFromPos(s string, pos int) (line uint, col uint, ok bool) {
+	// replace crlf with lf
+	s = strings.Replace(s, "\r\n", "\n", -1)
+	// pg docs: pos uses index 1 for the first character, and positions are measured in characters not bytes
+	runes := []rune(s)
+	if pos > len(runes) {
+		return 0, 0, false
+	}
+	sel := runes[:pos]
+	line = uint(runesCount(sel, newLine) + 1)
+	col = uint(pos - 1 - runesLastIndex(sel, newLine))
+	return line, col, true
+}
+
+const newLine = '\n'
+
+func runesCount(input []rune, target rune) int {
+	var count int
+	for _, r := range input {
+		if r == target {
+			count++
+		}
+	}
+	return count
+}
+
+func runesLastIndex(input []rune, target rune) int {
+	for i := len(input) - 1; i >= 0; i-- {
+		if input[i] == target {
+			return i
+		}
+	}
+	return -1
+}
+
 func (p *Postgres) SetVersion(version int, dirty bool) error {
-	tx, err := p.db.Begin()
+	tx, err := p.conn.BeginTx(context.Background(), &sql.TxOptions{})
 	if err != nil {
 		return &database.Error{OrigErr: err, Err: "transaction start failed"}
 	}
 
-	query := `TRUNCATE "` + p.config.MigrationsTable + `"`
-	if _, err := p.db.Exec(query); err != nil {
-		tx.Rollback()
+	query := `TRUNCATE ` + pq.QuoteIdentifier(p.config.MigrationsTable)
+	if _, err := tx.Exec(query); err != nil {
+		if errRollback := tx.Rollback(); errRollback != nil {
+			err = multierror.Append(err, errRollback)
+		}
 		return &database.Error{OrigErr: err, Query: []byte(query)}
 	}
 
 	if version >= 0 {
-		query = `INSERT INTO "` + p.config.MigrationsTable + `" (version, dirty) VALUES ($1, $2)`
-		if _, err := p.db.Exec(query, version, dirty); err != nil {
-			tx.Rollback()
+		query = `INSERT INTO ` + pq.QuoteIdentifier(p.config.MigrationsTable) + ` (version, dirty) VALUES ($1, $2)`
+		if _, err := tx.Exec(query, version, dirty); err != nil {
+			if errRollback := tx.Rollback(); errRollback != nil {
+				err = multierror.Append(err, errRollback)
+			}
 			return &database.Error{OrigErr: err, Query: []byte(query)}
 		}
 	}
@@ -197,8 +282,8 @@ func (p *Postgres) SetVersion(version int, dirty bool) error {
 }
 
 func (p *Postgres) Version() (version int, dirty bool, err error) {
-	query := `SELECT version, dirty FROM "` + p.config.MigrationsTable + `" LIMIT 1`
-	err = p.db.QueryRow(query).Scan(&version, &dirty)
+	query := `SELECT version, dirty FROM ` + pq.QuoteIdentifier(p.config.MigrationsTable) + ` LIMIT 1`
+	err = p.conn.QueryRowContext(context.Background(), query).Scan(&version, &dirty)
 	switch {
 	case err == sql.ErrNoRows:
 		return database.NilVersion, false, nil
@@ -216,14 +301,18 @@ func (p *Postgres) Version() (version int, dirty bool, err error) {
 	}
 }
 
-func (p *Postgres) Drop() error {
+func (p *Postgres) Drop() (err error) {
 	// select all tables in current schema
-	query := `SELECT table_name FROM information_schema.tables WHERE table_schema=(SELECT current_schema())`
-	tables, err := p.db.Query(query)
+	query := `SELECT table_name FROM information_schema.tables WHERE table_schema=(SELECT current_schema()) AND table_type='BASE TABLE'`
+	tables, err := p.conn.QueryContext(context.Background(), query)
 	if err != nil {
 		return &database.Error{OrigErr: err, Query: []byte(query)}
 	}
-	defer tables.Close()
+	defer func() {
+		if errClose := tables.Close(); errClose != nil {
+			err = multierror.Append(err, errClose)
+		}
+	}()
 
 	// delete one table after another
 	tableNames := make([]string, 0)
@@ -240,34 +329,38 @@ func (p *Postgres) Drop() error {
 	if len(tableNames) > 0 {
 		// delete one by one ...
 		for _, t := range tableNames {
-			query = `DROP TABLE IF EXISTS ` + t + ` CASCADE`
-			if _, err := p.db.Exec(query); err != nil {
+			query = `DROP TABLE IF EXISTS ` + pq.QuoteIdentifier(t) + ` CASCADE`
+			if _, err := p.conn.ExecContext(context.Background(), query); err != nil {
 				return &database.Error{OrigErr: err, Query: []byte(query)}
 			}
 		}
-		if err := p.ensureVersionTable(); err != nil {
-			return err
+	}
+
+	return nil
+}
+
+// ensureVersionTable checks if versions table exists and, if not, creates it.
+// Note that this function locks the database, which deviates from the usual
+// convention of "caller locks" in the Postgres type.
+func (p *Postgres) ensureVersionTable() (err error) {
+	if err = p.Lock(); err != nil {
+		return err
+	}
+
+	defer func() {
+		if e := p.Unlock(); e != nil {
+			if err == nil {
+				err = e
+			} else {
+				err = multierror.Append(err, e)
+			}
 		}
+	}()
+
+	query := `CREATE TABLE IF NOT EXISTS ` + pq.QuoteIdentifier(p.config.MigrationsTable) + ` (version bigint not null primary key, dirty boolean not null)`
+	if _, err = p.conn.ExecContext(context.Background(), query); err != nil {
+		return &database.Error{OrigErr: err, Query: []byte(query)}
 	}
 
 	return nil
 }
-
-func (p *Postgres) ensureVersionTable() error {
-	// check if migration table exists
-	var count int
-	query := `SELECT COUNT(1) FROM information_schema.tables WHERE table_name = $1 AND table_schema = (SELECT current_schema()) LIMIT 1`
-	if err := p.db.QueryRow(query, p.config.MigrationsTable).Scan(&count); err != nil {
-		return &database.Error{OrigErr: err, Query: []byte(query)}
-	}
-	if count == 1 {
-		return nil
-	}
-
-	// if not, create the empty migration table
-	query = `CREATE TABLE "` + p.config.MigrationsTable + `" (version bigint not null primary key, dirty boolean not null)`
-	if _, err := p.db.Exec(query); err != nil {
-		return &database.Error{OrigErr: err, Query: []byte(query)}
-	}
-	return nil
-}

database/postgres/postgres_test.go

@@ -3,148 +3,499 @@ package postgres
 // error codes https://github.com/lib/pq/blob/master/error.go
 
 import (
-	"bytes"
+	"context"
 	"database/sql"
+	sqldriver "database/sql/driver"
 	"fmt"
+	"log"
+
+	"github.com/golang-migrate/migrate/v4"
 	"io"
+	"strconv"
+	"strings"
+	"sync"
 	"testing"
 
-	"github.com/lib/pq"
-	dt "github.com/mattes/migrate/database/testing"
-	mt "github.com/mattes/migrate/testing"
+	"github.com/dhui/dktest"
+
+	dt "github.com/golang-migrate/migrate/v4/database/testing"
+	"github.com/golang-migrate/migrate/v4/dktesting"
+	_ "github.com/golang-migrate/migrate/v4/source/file"
 )
 
-var versions = []mt.Version{
-	{Image: "postgres:9.6"},
-	{Image: "postgres:9.5"},
-	{Image: "postgres:9.4"},
-	{Image: "postgres:9.3"},
-	{Image: "postgres:9.2"},
+var (
+	opts = dktest.Options{PortRequired: true, ReadyFunc: isReady}
+	// Supported versions: https://www.postgresql.org/support/versioning/
+	specs = []dktesting.ContainerSpec{
+		{ImageName: "postgres:9.4", Options: opts},
+		{ImageName: "postgres:9.5", Options: opts},
+		{ImageName: "postgres:9.6", Options: opts},
+		{ImageName: "postgres:10", Options: opts},
+		{ImageName: "postgres:11", Options: opts},
+	}
+)
+
+func pgConnectionString(host, port string) string {
+	return fmt.Sprintf("postgres://postgres@%s:%s/postgres?sslmode=disable", host, port)
 }
 
-func isReady(i mt.Instance) bool {
-	db, err := sql.Open("postgres", fmt.Sprintf("postgres://postgres@%v:%v/postgres?sslmode=disable", i.Host(), i.Port()))
+func isReady(ctx context.Context, c dktest.ContainerInfo) bool {
+	ip, port, err := c.FirstPort()
 	if err != nil {
 		return false
 	}
-	defer db.Close()
-	err = db.Ping()
-	if err == io.EOF {
-		return false
 
-	} else if e, ok := err.(*pq.Error); ok {
-		if e.Code.Name() == "cannot_connect_now" {
-			return false
+	db, err := sql.Open("postgres", pgConnectionString(ip, port))
+	if err != nil {
+		return false
+	}
+	defer func() {
+		if err := db.Close(); err != nil {
+			log.Println("close error:", err)
 		}
+	}()
+	if err = db.PingContext(ctx); err != nil {
+		switch err {
+		case sqldriver.ErrBadConn, io.EOF:
+			return false
+		default:
+			log.Println(err)
+		}
+		return false
 	}
 
 	return true
 }
 
 func Test(t *testing.T) {
-	mt.ParallelTest(t, versions, isReady,
-		func(t *testing.T, i mt.Instance) {
-			p := &Postgres{}
-			addr := fmt.Sprintf("postgres://postgres@%v:%v/postgres?sslmode=disable", i.Host(), i.Port())
-			d, err := p.Open(addr)
-			if err != nil {
-				t.Fatalf("%v", err)
+	dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
+		ip, port, err := c.FirstPort()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		addr := pgConnectionString(ip, port)
+		p := &Postgres{}
+		d, err := p.Open(addr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer func() {
+			if err := d.Close(); err != nil {
+				t.Error(err)
 			}
-			dt.Test(t, d, []byte("SELECT 1"))
-		})
+		}()
+		dt.Test(t, d, []byte("SELECT 1"))
+	})
+}
+
+func TestMigrate(t *testing.T) {
+	dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
+		ip, port, err := c.FirstPort()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		addr := pgConnectionString(ip, port)
+		p := &Postgres{}
+		d, err := p.Open(addr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer func() {
+			if err := d.Close(); err != nil {
+				t.Error(err)
+			}
+		}()
+		m, err := migrate.NewWithDatabaseInstance("file://./examples/migrations", "postgres", d)
+		if err != nil {
+			t.Fatal(err)
+		}
+		dt.TestMigrate(t, m, []byte("SELECT 1"))
+	})
 }
 
 func TestMultiStatement(t *testing.T) {
-	mt.ParallelTest(t, versions, isReady,
-		func(t *testing.T, i mt.Instance) {
-			p := &Postgres{}
-			addr := fmt.Sprintf("postgres://postgres@%v:%v/postgres?sslmode=disable", i.Host(), i.Port())
-			d, err := p.Open(addr)
-			if err != nil {
-				t.Fatalf("%v", err)
-			}
-			if err := d.Run(bytes.NewReader([]byte("CREATE TABLE foo (foo text); CREATE TABLE bar (bar text);"))); err != nil {
-				t.Fatalf("expected err to be nil, got %v", err)
-			}
+	dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
+		ip, port, err := c.FirstPort()
+		if err != nil {
+			t.Fatal(err)
+		}
 
-			// make sure second table exists
-			var exists bool
-			if err := d.(*Postgres).db.QueryRow("SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'bar' AND table_schema = (SELECT current_schema()))").Scan(&exists); err != nil {
-				t.Fatal(err)
+		addr := pgConnectionString(ip, port)
+		p := &Postgres{}
+		d, err := p.Open(addr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer func() {
+			if err := d.Close(); err != nil {
+				t.Error(err)
 			}
-			if !exists {
-				t.Fatalf("expected table bar to exist")
+		}()
+		if err := d.Run(strings.NewReader("CREATE TABLE foo (foo text); CREATE TABLE bar (bar text);")); err != nil {
+			t.Fatalf("expected err to be nil, got %v", err)
+		}
+
+		// make sure second table exists
+		var exists bool
+		if err := d.(*Postgres).conn.QueryRowContext(context.Background(), "SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'bar' AND table_schema = (SELECT current_schema()))").Scan(&exists); err != nil {
+			t.Fatal(err)
+		}
+		if !exists {
+			t.Fatalf("expected table bar to exist")
+		}
+	})
+}
+
+func TestErrorParsing(t *testing.T) {
+	dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
+		ip, port, err := c.FirstPort()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		addr := pgConnectionString(ip, port)
+		p := &Postgres{}
+		d, err := p.Open(addr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer func() {
+			if err := d.Close(); err != nil {
+				t.Error(err)
 			}
-		})
+		}()
+
+		wantErr := `migration failed: syntax error at or near "TABLEE" (column 37) in line 1: CREATE TABLE foo ` +
+			`(foo text); CREATE TABLEE bar (bar text); (details: pq: syntax error at or near "TABLEE")`
+		if err := d.Run(strings.NewReader("CREATE TABLE foo (foo text); CREATE TABLEE bar (bar text);")); err == nil {
+			t.Fatal("expected err but got nil")
+		} else if err.Error() != wantErr {
+			t.Fatalf("expected '%s' but got '%s'", wantErr, err.Error())
+		}
+	})
 }
 
 func TestFilterCustomQuery(t *testing.T) {
-	mt.ParallelTest(t, versions, isReady,
-		func(t *testing.T, i mt.Instance) {
-			p := &Postgres{}
-			addr := fmt.Sprintf("postgres://postgres@%v:%v/postgres?sslmode=disable&x-custom=foobar", i.Host(), i.Port())
-			_, err := p.Open(addr)
-			if err != nil {
-				t.Fatalf("%v", err)
+	dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
+		ip, port, err := c.FirstPort()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		addr := fmt.Sprintf("postgres://postgres@%v:%v/postgres?sslmode=disable&x-custom=foobar", ip, port)
+		p := &Postgres{}
+		d, err := p.Open(addr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer func() {
+			if err := d.Close(); err != nil {
+				t.Error(err)
 			}
-		})
+		}()
+	})
 }
 
 func TestWithSchema(t *testing.T) {
-	mt.ParallelTest(t, versions, isReady,
-		func(t *testing.T, i mt.Instance) {
-			p := &Postgres{}
-			addr := fmt.Sprintf("postgres://postgres@%v:%v/postgres?sslmode=disable", i.Host(), i.Port())
-			d, err := p.Open(addr)
-			if err != nil {
-				t.Fatalf("%v", err)
-			}
+	dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
+		ip, port, err := c.FirstPort()
+		if err != nil {
+			t.Fatal(err)
+		}
 
-			// create foobar schema
-			if err := d.Run(bytes.NewReader([]byte("CREATE SCHEMA foobar AUTHORIZATION postgres"))); err != nil {
-				t.Fatal(err)
-			}
-			if err := d.SetVersion(1, false); err != nil {
+		addr := pgConnectionString(ip, port)
+		p := &Postgres{}
+		d, err := p.Open(addr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer func() {
+			if err := d.Close(); err != nil {
 				t.Fatal(err)
 			}
+		}()
 
-			// re-connect using that schema
-			d2, err := p.Open(fmt.Sprintf("postgres://postgres@%v:%v/postgres?sslmode=disable&search_path=foobar", i.Host(), i.Port()))
-			if err != nil {
-				t.Fatalf("%v", err)
-			}
+		// create foobar schema
+		if err := d.Run(strings.NewReader("CREATE SCHEMA foobar AUTHORIZATION postgres")); err != nil {
+			t.Fatal(err)
+		}
+		if err := d.SetVersion(1, false); err != nil {
+			t.Fatal(err)
+		}
 
-			version, _, err := d2.Version()
-			if err != nil {
+		// re-connect using that schema
+		d2, err := p.Open(fmt.Sprintf("postgres://postgres@%v:%v/postgres?sslmode=disable&search_path=foobar", ip, port))
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer func() {
+			if err := d2.Close(); err != nil {
 				t.Fatal(err)
 			}
-			if version != -1 {
-				t.Fatal("expected NilVersion")
-			}
+		}()
 
-			// now update version and compare
-			if err := d2.SetVersion(2, false); err != nil {
-				t.Fatal(err)
-			}
-			version, _, err = d2.Version()
-			if err != nil {
-				t.Fatal(err)
-			}
-			if version != 2 {
-				t.Fatal("expected version 2")
-			}
+		version, _, err := d2.Version()
+		if err != nil {
+			t.Fatal(err)
+		}
+		if version != -1 {
+			t.Fatal("expected NilVersion")
+		}
 
-			// meanwhile, the public schema still has the other version
-			version, _, err = d.Version()
-			if err != nil {
-				t.Fatal(err)
+		// now update version and compare
+		if err := d2.SetVersion(2, false); err != nil {
+			t.Fatal(err)
+		}
+		version, _, err = d2.Version()
+		if err != nil {
+			t.Fatal(err)
+		}
+		if version != 2 {
+			t.Fatal("expected version 2")
+		}
+
+		// meanwhile, the public schema still has the other version
+		version, _, err = d.Version()
+		if err != nil {
+			t.Fatal(err)
+		}
+		if version != 1 {
+			t.Fatal("expected version 2")
+		}
+	})
+}
+
+func TestParallelSchema(t *testing.T) {
+	dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
+		ip, port, err := c.FirstPort()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		addr := pgConnectionString(ip, port)
+		p := &Postgres{}
+		d, err := p.Open(addr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer func() {
+			if err := d.Close(); err != nil {
+				t.Error(err)
 			}
-			if version != 1 {
-				t.Fatal("expected version 2")
+		}()
+
+		// create foo and bar schemas
+		if err := d.Run(strings.NewReader("CREATE SCHEMA foo AUTHORIZATION postgres")); err != nil {
+			t.Fatal(err)
+		}
+		if err := d.Run(strings.NewReader("CREATE SCHEMA bar AUTHORIZATION postgres")); err != nil {
+			t.Fatal(err)
+		}
+
+		// re-connect using that schemas
+		dfoo, err := p.Open(fmt.Sprintf("postgres://postgres@%v:%v/postgres?sslmode=disable&search_path=foo", ip, port))
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer func() {
+			if err := dfoo.Close(); err != nil {
+				t.Error(err)
 			}
-		})
+		}()
+
+		dbar, err := p.Open(fmt.Sprintf("postgres://postgres@%v:%v/postgres?sslmode=disable&search_path=bar", ip, port))
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer func() {
+			if err := dbar.Close(); err != nil {
+				t.Error(err)
+			}
+		}()
+
+		if err := dfoo.Lock(); err != nil {
+			t.Fatal(err)
+		}
+
+		if err := dbar.Lock(); err != nil {
+			t.Fatal(err)
+		}
+
+		if err := dbar.Unlock(); err != nil {
+			t.Fatal(err)
+		}
+
+		if err := dfoo.Unlock(); err != nil {
+			t.Fatal(err)
+		}
+	})
 }
 
 func TestWithInstance(t *testing.T) {
 
 }
+
+func TestPostgres_Lock(t *testing.T) {
+	dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
+		ip, port, err := c.FirstPort()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		addr := pgConnectionString(ip, port)
+		p := &Postgres{}
+		d, err := p.Open(addr)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		dt.Test(t, d, []byte("SELECT 1"))
+
+		ps := d.(*Postgres)
+
+		err = ps.Lock()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		err = ps.Unlock()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		err = ps.Lock()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		err = ps.Unlock()
+		if err != nil {
+			t.Fatal(err)
+		}
+	})
+}
+
+func TestWithInstance_Concurrent(t *testing.T) {
+	dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
+		ip, port, err := c.FirstPort()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// The number of concurrent processes running WithInstance
+		const concurrency = 30
+
+		// We can instantiate a single database handle because it is
+		// actually a connection pool, and so, each of the below go
+		// routines will have a high probability of using a separate
+		// connection, which is something we want to exercise.
+		db, err := sql.Open("postgres", pgConnectionString(ip, port))
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer func() {
+			if err := db.Close(); err != nil {
+				t.Error(err)
+			}
+		}()
+
+		db.SetMaxIdleConns(concurrency)
+		db.SetMaxOpenConns(concurrency)
+
+		var wg sync.WaitGroup
+		defer wg.Wait()
+
+		wg.Add(concurrency)
+		for i := 0; i < concurrency; i++ {
+			go func(i int) {
+				defer wg.Done()
+				_, err := WithInstance(db, &Config{})
+				if err != nil {
+					t.Errorf("process %d error: %s", i, err)
+				}
+			}(i)
+		}
+	})
+}
+func Test_computeLineFromPos(t *testing.T) {
+	testcases := []struct {
+		pos      int
+		wantLine uint
+		wantCol  uint
+		input    string
+		wantOk   bool
+	}{
+		{
+			15, 2, 6, "SELECT *\nFROM foo", true, // foo table does not exists
+		},
+		{
+			16, 3, 6, "SELECT *\n\nFROM foo", true, // foo table does not exists, empty line
+		},
+		{
+			25, 3, 7, "SELECT *\nFROM foo\nWHERE x", true, // x column error
+		},
+		{
+			27, 5, 7, "SELECT *\n\nFROM foo\n\nWHERE x", true, // x column error, empty lines
+		},
+		{
+			10, 2, 1, "SELECT *\nFROMM foo", true, // FROMM typo
+		},
+		{
+			11, 3, 1, "SELECT *\n\nFROMM foo", true, // FROMM typo, empty line
+		},
+		{
+			17, 2, 8, "SELECT *\nFROM foo", true, // last character
+		},
+		{
+			18, 0, 0, "SELECT *\nFROM foo", false, // invalid position
+		},
+	}
+	for i, tc := range testcases {
+		t.Run("tc"+strconv.Itoa(i), func(t *testing.T) {
+			run := func(crlf bool, nonASCII bool) {
+				var name string
+				if crlf {
+					name = "crlf"
+				} else {
+					name = "lf"
+				}
+				if nonASCII {
+					name += "-nonascii"
+				} else {
+					name += "-ascii"
+				}
+				t.Run(name, func(t *testing.T) {
+					input := tc.input
+					if crlf {
+						input = strings.Replace(input, "\n", "\r\n", -1)
+					}
+					if nonASCII {
+						input = strings.Replace(input, "FROM", "FRÖM", -1)
+					}
+					gotLine, gotCol, gotOK := computeLineFromPos(input, tc.pos)
+
+					if tc.wantOk {
+						t.Logf("pos %d, want %d:%d, %#v", tc.pos, tc.wantLine, tc.wantCol, input)
+					}
+
+					if gotOK != tc.wantOk {
+						t.Fatalf("expected ok %v but got %v", tc.wantOk, gotOK)
+					}
+					if gotLine != tc.wantLine {
+						t.Fatalf("expected line %d but got %d", tc.wantLine, gotLine)
+					}
+					if gotCol != tc.wantCol {
+						t.Fatalf("expected col %d but got %d", tc.wantCol, gotCol)
+					}
+				})
+			}
+			run(false, false)
+			run(true, false)
+			run(false, true)
+			run(true, true)
+		})
+	}
+
+}

database/ql/ql.go

@@ -3,6 +3,7 @@ package ql
 import (
 	"database/sql"
 	"fmt"
+	"github.com/hashicorp/go-multierror"
 	"io"
 	"io/ioutil"
 	"strings"
@@ -10,8 +11,8 @@ import (
 	nurl "net/url"
 
 	_ "github.com/cznic/ql/driver"
-	"github.com/mattes/migrate"
-	"github.com/mattes/migrate/database"
+	"github.com/golang-migrate/migrate/v4"
+	"github.com/golang-migrate/migrate/v4/database"
 )
 
 func init() {
@@ -46,6 +47,7 @@ func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) {
 	if err := instance.Ping(); err != nil {
 		return nil, err
 	}
+
 	if len(config.MigrationsTable) == 0 {
 		config.MigrationsTable = DefaultMigrationsTable
 	}
@@ -59,7 +61,25 @@ func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) {
 	}
 	return mx, nil
 }
-func (m *Ql) ensureVersionTable() error {
+
+// ensureVersionTable checks if versions table exists and, if not, creates it.
+// Note that this function locks the database, which deviates from the usual
+// convention of "caller locks" in the Ql type.
+func (m *Ql) ensureVersionTable() (err error) {
+	if err = m.Lock(); err != nil {
+		return err
+	}
+
+	defer func() {
+		if e := m.Unlock(); e != nil {
+			if err == nil {
+				err = e
+			} else {
+				err = multierror.Append(err, e)
+			}
+		}
+	}()
+
 	tx, err := m.db.Begin()
 	if err != nil {
 		return err
@@ -105,13 +125,17 @@ func (m *Ql) Open(url string) (database.Driver, error) {
 func (m *Ql) Close() error {
 	return m.db.Close()
 }
-func (m *Ql) Drop() error {
+func (m *Ql) Drop() (err error) {
 	query := `SELECT Name FROM __Table`
 	tables, err := m.db.Query(query)
 	if err != nil {
 		return &database.Error{OrigErr: err, Query: []byte(query)}
 	}
-	defer tables.Close()
+	defer func() {
+		if errClose := tables.Close(); errClose != nil {
+			err = multierror.Append(err, errClose)
+		}
+	}()
 	tableNames := make([]string, 0)
 	for tables.Next() {
 		var tableName string
@@ -119,7 +143,7 @@ func (m *Ql) Drop() error {
 			return err
 		}
 		if len(tableName) > 0 {
-			if strings.HasPrefix(tableName, "__") == false {
+			if !strings.HasPrefix(tableName, "__") {
 				tableNames = append(tableNames, tableName)
 			}
 		}
@@ -132,9 +156,6 @@ func (m *Ql) Drop() error {
 				return &database.Error{OrigErr: err, Query: []byte(query)}
 			}
 		}
-		if err := m.ensureVersionTable(); err != nil {
-			return err
-		}
 	}
 
 	return nil
@@ -168,7 +189,9 @@ func (m *Ql) executeQuery(query string) error {
 		return &database.Error{OrigErr: err, Err: "transaction start failed"}
 	}
 	if _, err := tx.Exec(query); err != nil {
-		tx.Rollback()
+		if errRollback := tx.Rollback(); errRollback != nil {
+			err = multierror.Append(err, errRollback)
+		}
 		return &database.Error{OrigErr: err, Query: []byte(query)}
 	}
 	if err := tx.Commit(); err != nil {
@@ -190,7 +213,9 @@ func (m *Ql) SetVersion(version int, dirty bool) error {
 	if version >= 0 {
 		query := fmt.Sprintf(`INSERT INTO %s (version, dirty) VALUES (%d, %t)`, m.config.MigrationsTable, version, dirty)
 		if _, err := tx.Exec(query); err != nil {
-			tx.Rollback()
+			if errRollback := tx.Rollback(); errRollback != nil {
+				err = multierror.Append(err, errRollback)
+			}
 			return &database.Error{OrigErr: err, Query: []byte(query)}
 		}
 	}

database/ql/ql_test.go

@@ -9,9 +9,9 @@ import (
 	"testing"
 
 	_ "github.com/cznic/ql/driver"
-	"github.com/mattes/migrate"
-	dt "github.com/mattes/migrate/database/testing"
-	_ "github.com/mattes/migrate/source/file"
+	"github.com/golang-migrate/migrate/v4"
+	dt "github.com/golang-migrate/migrate/v4/database/testing"
+	_ "github.com/golang-migrate/migrate/v4/source/file"
 )
 
 func Test(t *testing.T) {
@@ -20,14 +20,16 @@ func Test(t *testing.T) {
 		return
 	}
 	defer func() {
-		os.RemoveAll(dir)
+		if err := os.RemoveAll(dir); err != nil {
+			t.Fatal(err)
+		}
 	}()
-	fmt.Printf("DB path : %s\n", filepath.Join(dir, "ql.db"))
+	t.Logf("DB path : %s\n", filepath.Join(dir, "ql.db"))
 	p := &Ql{}
 	addr := fmt.Sprintf("ql://%s", filepath.Join(dir, "ql.db"))
 	d, err := p.Open(addr)
 	if err != nil {
-		t.Fatalf("%v", err)
+		t.Fatal(err)
 	}
 
 	db, err := sql.Open("ql", filepath.Join(dir, "ql.db"))
@@ -40,23 +42,40 @@ func Test(t *testing.T) {
 		}
 	}()
 	dt.Test(t, d, []byte("CREATE TABLE t (Qty int, Name string);"))
+}
+
+func TestMigrate(t *testing.T) {
+	dir, err := ioutil.TempDir("", "ql-driver-test")
+	if err != nil {
+		return
+	}
+	defer func() {
+		if err := os.RemoveAll(dir); err != nil {
+			t.Error(err)
+		}
+	}()
+	t.Logf("DB path : %s\n", filepath.Join(dir, "ql.db"))
+
+	db, err := sql.Open("ql", filepath.Join(dir, "ql.db"))
+	if err != nil {
+		return
+	}
+	defer func() {
+		if err := db.Close(); err != nil {
+			return
+		}
+	}()
+
 	driver, err := WithInstance(db, &Config{})
 	if err != nil {
-		t.Fatalf("%v", err)
-	}
-	if err := d.Drop(); err != nil {
 		t.Fatal(err)
 	}
 
 	m, err := migrate.NewWithDatabaseInstance(
-		"file://./migration",
+		"file://./examples/migrations",
 		"ql", driver)
 	if err != nil {
-		t.Fatalf("%v", err)
-	}
-	fmt.Println("UP")
-	err = m.Up()
-	if err != nil {
-		t.Fatalf("%v", err)
+		t.Fatal(err)
 	}
+	dt.TestMigrate(t, m, []byte("CREATE TABLE t (Qty int, Name string);"))
 }

database/redshift/README.md

@@ -1,6 +1,21 @@
-Redshift
-===
+# Redshift
 
-This provides a Redshift driver for migrations.  It is used whenever the URL of the database starts with `redshift://`.
+`redshift://user:password@host:port/dbname?query`
+
+| URL Query  | WithInstance Config | Description |
+|------------|---------------------|-------------|
+| `x-migrations-table` | `MigrationsTable` | Name of the migrations table |
+| `dbname` | `DatabaseName` | The name of the database to connect to |
+| `search_path` | | This variable specifies the order in which schemas are searched when an object is referenced by a simple name with no schema specified. |
+| `user` | | The user to sign in as |
+| `password` | | The user's password | 
+| `host` | | The host to connect to. Values that start with / are for unix domain sockets. (default is localhost) |
+| `port` | | The port to bind to. (default is 5439) |
+| `fallback_application_name` | | An application_name to fall back to if one isn't provided. |
+| `connect_timeout` | | Maximum wait for connection, in seconds. Zero or not specified means wait indefinitely. |
+| `sslcert` | | Cert file location. The file must contain PEM encoded data. |
+| `sslkey` | | Key file location. The file must contain PEM encoded data. |
+| `sslrootcert` | | The location of the root certificate file. The file must contain PEM encoded data. | 
+| `sslmode` | | Whether or not to use SSL (disable\|require\|verify-ca\|verify-full) |
 
 Redshift is PostgreSQL compatible but has some specific features (or lack thereof) that require slightly different behavior.

database/redshift/examples/migrations/1085649617_create_users_table.down.sql

@@ -0,0 +1 @@
+DROP TABLE IF EXISTS users;

database/redshift/examples/migrations/1085649617_create_users_table.up.sql

@@ -0,0 +1,5 @@
+CREATE TABLE users (
+  user_id integer unique,
+  name    varchar(40),
+  email   varchar(40)
+);

database/redshift/examples/migrations/1185749658_add_city_to_users.down.sql

@@ -0,0 +1 @@
+ALTER TABLE users DROP COLUMN IF EXISTS city;

database/redshift/examples/migrations/1185749658_add_city_to_users.up.sql

@@ -0,0 +1,3 @@
+ALTER TABLE users ADD COLUMN city varchar(100);
+
+