diff --git a/.github/workflows/rubocop.yml b/.github/workflows/rubocop.yml new file mode 100644 index 0000000..fc5f115 --- /dev/null +++ b/.github/workflows/rubocop.yml @@ -0,0 +1,29 @@ +name: Run Rubocop +on: + push: + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Setup Gemfile + run: | + touch .enable_test + + - name: Setup Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: 3.1 + bundler-cache: true + + - name: Install Ruby dependencies + run: | + bundle install --jobs 4 --retry 3 + + - name: Run RuboCop + run: | + bundle exec rubocop -S diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..6b40a38 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,86 @@ +name: Tests +on: + push: + pull_request: + +jobs: + test: + name: ruby-${{ matrix.ruby }} + runs-on: ubuntu-latest + + strategy: + matrix: + ruby: ['3.1', '3.2', '3.3', '3.4'] + fail-fast: false + + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: somename + ports: + - 5432:5432 + + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + mysql: + image: mysql:8.0 + env: + MYSQL_ROOT_PASSWORD: 'BestPasswordEver' + MYSQL_DATABASE: somename + ports: + - 3306:3306 + options: >- + --health-cmd="mysqladmin ping" + --health-interval=10s + --health-timeout=5s + --health-retries=3 + + steps: + - name: Verify MySQL connection from host + run: | + mysql --host 127.0.0.1 --port ${{ job.services.mysql.ports[3306] }} -uroot -pBestPasswordEver -e "SHOW DATABASES" + + - name: Checkout git + uses: actions/checkout@v2 + + - name: Update package archives + run: sudo apt-get update --yes --quiet + + - name: Install package dependencies + run: > + sudo apt-get install --yes --quiet + build-essential + cmake + libicu-dev + libpq-dev + libmysqlclient-dev + + - name: Setup Gemfile + run: | + touch .enable_test + + - name: Setup Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby }} + bundler-cache: true + + - name: Install Ruby dependencies + run: | + bundle install --jobs=4 --retry=3 + + - name: Run tests + env: + MYSQL2POSTGRES_ENV: test + MYSQL_PORT: ${{ job.services.mysql.ports[3306] }} + run: bundle exec rake test + + - name: Run build gem file + run: gem build mysql2postgres.gemspec diff --git a/.gitignore b/.gitignore index 42965bc..e85c824 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.DS_Store *.gem *.sql .bundle @@ -6,3 +7,5 @@ config/database.yml Gemfile.lock pkg test/fixtures/test*.sql +.enable_dev +.enable_test diff --git a/.rubocop.yml b/.rubocop.yml new file mode 100644 index 0000000..09884eb --- /dev/null +++ b/.rubocop.yml @@ -0,0 +1,58 @@ +plugins: + - rubocop-performance + +AllCops: + TargetRubyVersion: 3.1 + NewCops: enable + +Metrics/AbcSize: + Enabled: false + +Metrics/BlockLength: + Enabled: false + +Metrics/ClassLength: + Enabled: false + +Metrics/CyclomaticComplexity: + Enabled: false + +Layout/LineLength: + Max: 140 + +Metrics/MethodLength: + Enabled: false + +Metrics/ModuleLength: + Enabled: false + +Metrics/PerceivedComplexity: + Max: 35 + +Performance/ChainArrayAllocation: + Enabled: true + +Style/Documentation: + Enabled: false + +Style/ExpandPathArguments: + Enabled: false + +Metrics/ParameterLists: + Enabled: true + CountKeywordArgs: false + +Style/MethodCallWithArgsParentheses: + Enabled: true + AllowParenthesesInMultilineCall: true + AllowParenthesesInChaining: true + EnforcedStyle: omit_parentheses + +Naming/FileName: + Enabled: true + Exclude: + - Rakefile + +Style/SafeNavigationChainLength: + Enabled: true + Max: 3 diff --git a/.ruby-version b/.ruby-version new file mode 100644 index 0000000..23887f6 --- /dev/null +++ b/.ruby-version @@ -0,0 +1 @@ +3.1.7 diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index c686469..0000000 --- a/.travis.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- -os: linux -dist: xenial - -language: ruby -cache: bundler -rvm: - - 2.7.1 - - 2.6.6 - - 2.5.8 - - 2.4.10 - - 2.3.8 - - 2.2.10 - - 2.1.10 - - ruby-head - - jruby-9.2.13.0 - -jobs: - allow_failures: - - rvm: ruby-head diff --git a/Gemfile b/Gemfile index 6693ca0..a08b426 100644 --- a/Gemfile +++ b/Gemfile @@ -1,22 +1,18 @@ -source 'https://rubygems.org/' - -gem 'rake', '~> 10.3' -gem 'mysql-pr', '~> 2.9' -gem 'postgres-pr', '~> 0.6' +# frozen_string_literal: true -platforms :jruby do - gem 'activerecord' - gem 'jdbc-postgres' - gem 'activerecord-jdbc-adapter' - gem 'activerecord-jdbcpostgresql-adapter' -end +source 'https://rubygems.org/' +gemspec -platforms :mri do - gem 'pg', '~> 0.18' +if File.file? File.expand_path './.enable_dev', __dir__ + group :development do + gem 'debug' + end end -gem 'test-unit', '~> 2.1' - -group :test do - gem 'jeweler', '~> 2.0' +if File.file? File.expand_path './.enable_test', __dir__ + group :development, :test do + gem 'rubocop', require: false + gem 'rubocop-performance', require: false + gem 'test-unit', '~> 3.6.8' + end end diff --git a/README.md b/README.md index 4dd5844..58e66f5 100644 --- a/README.md +++ b/README.md @@ -1,113 +1,49 @@ -# mysql-to-postgres - MySQL to PostgreSQL Data Translation +# mysql2postgres - MySQL to PostgreSQL Data Translation -[![Build Status](https://travis-ci.org/maxlapshin/mysql2postgres.svg)](https://travis-ci.org/maxlapshin/mysql2postgres) -[![Dependency Status](https://gemnasium.com/maxlapshin/mysql2postgres.svg)](https://gemnasium.com/maxlapshin/mysql2postgres) +[![Run Linters](https://github.com/AlphaNodes/mysql2postgres/workflows/Run%20Rubocop/badge.svg)](https://github.com/AlphaNodes/mysql2postgres/actions/workflows/rubocop.yml) [![Run Tests](https://github.com/AlphaNodes/mysql2postgres/workflows/Tests/badge.svg)](https://github.com/AlphaNodes/mysql2postgres/actions/workflows/tests.yml) -MRI or jruby supported. The minimum Ruby version supported in `master` branch is `2.1.7`, -and the next release will have the same requirement. +Convert MySQL database to PostgreSQL database. -With a bit of a modified rails `database.yml` configuration, you can integrate `mysql-to-postgres`into a project. +## Requirements -## Installation - -**Currently failing, see #81...** +- Ruby `>= 2.7` (only maintained ruby versions are supported) -### Via RubyGems - -```sh -gem install mysqltopostgres -``` +## Installation -### From source +Add Gem to your Gemfile: -```sh -git clone https://github.com/maxlapshin/mysql2postgres.git -cd mysql2postgres -bundle install -gem build mysqltopostgres.gemspec -sudo gem install mysqltopostgres-0.3.1.gem +```ruby +gem 'mysql2postgres' ``` -## Sample Configuration +## Configuration Configuration is written in [YAML format](http://www.yaml.org/ "YAML Ain't Markup Language") and passed as the first argument on the command line. -```yaml -default: &default - adapter: jdbcpostgresql - encoding: unicode - pool: 4 - username: terrapotamus - password: default - host: 127.0.0.1 - -development: &development - <<: *default - database: default_development - -test: &test - <<: *default - database: default_test - -production: &production - <<: *default - database: default_production - -mysql_data_source: &pii - host: localhost - port: 3306 - socket: /tmp/mysqld.sock - username: username - password: default - database: awesome_possum - -mysql2psql: - mysql: - <<: *pii - - destination: - production: - <<: *production - test: - <<: *test - development: - <<: *development - - tables: - - countries - - samples - - universes - - variable_groups - - variables - - sample_variables - - # If suppress_data is true, only the schema definition will be exported/migrated, and not the data - suppress_data: false - - # If suppress_ddl is true, only the data will be exported/imported, and not the schema - suppress_ddl: true - - # If force_truncate is true, forces a table truncate before table loading - force_truncate: false - - preserve_order: true - - remove_dump_file: true - - dump_file_directory: /tmp - - report_status: json # false, json, xml - - # If clear_schema is true, the public schema will be recreated before conversion - # The import will fail if both clear_schema and suppress_ddl are true. - clear_schema: false -``` +Configuration file has be provided with config/database.yml, see [config/default.database.yml](config/default.database.yml) for an example and for configuration information. + +## Usage -Please note that the MySQL connection will be using socket in case the host is not defined (`nil`) or it is `'localhost'`. +After providing settings, start migration with -## Testing +```sh +# set destination to use +MYSQL2POSTGRES_ENV=test +# use can also use (MYSQL2POSTGRES_ENV is used, if both are defined) +RAILS_ENV=test + +# with default configuration, which use config/database.yml +bundle exec mysql2postgres +# OR with specified configuration file +bundle exec mysql2postgres /home/you/mysql2postgres.yml +``` +## Tests + +```sh +rake test +``` ## License diff --git a/Rakefile b/Rakefile index 034abe6..d5e28b4 100644 --- a/Rakefile +++ b/Rakefile @@ -1,53 +1,19 @@ +# frozen_string_literal: true + require 'rubygems' require 'rake' -require_relative 'lib/mysql2psql/version' - -begin - require 'jeweler' - Jeweler::Tasks.new do |gem| - gem.name = 'mysql2psql' - gem.version = Mysql2psql::Version::STRING - gem.summary = %(Tool for converting mysql database to postgresql) - gem.description = %{It can create postgresql dump from mysql database or directly load data from mysql to - postgresql (at about 100 000 records per minute). Translates most data types and indexes.} - gem.email = 'gallagher.paul@gmail.com' - gem.homepage = 'http://github.com/tardate/mysql2postgresql' - gem.authors = [ - 'Max Lapshin ', - 'Anton Ageev ', - 'Samuel Tribehou ', - 'Marco Nenciarini ', - 'James Nobis ', - 'quel ', - 'Holger Amann ', - 'Maxim Dobriakov ', - 'Michael Kimsal ', - 'Jacob Coby ', - 'Neszt Tibor ', - 'Miroslav Kratochvil ', - 'Paul Gallagher ', - 'Juga Paazmaya ' - ] - gem.add_dependency 'mysql-pr', '~> 2.9' - gem.add_dependency 'postgres-pr', '~> 0.6' - gem.add_development_dependency 'test-unit', '~> 2.1' - # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings - end - Jeweler::GemcutterTasks.new -rescue LoadError - puts 'Jeweler (or a dependency) not available. Install it with: gem install jeweler' -end +require_relative 'lib/mysql2postgres/version' require 'rake/testtask' namespace :test do - Rake::TestTask.new(:units) do |test| + Rake::TestTask.new :units do |test| test.libs << 'lib' << 'test/lib' test.pattern = 'test/units/*test.rb' test.verbose = true end - Rake::TestTask.new(:integration) do |test| + Rake::TestTask.new :integration do |test| test.libs << 'lib' << 'test/lib' test.pattern = 'test/integration/*test.rb' test.verbose = true @@ -57,7 +23,7 @@ end desc 'Run all tests' task :test do Rake::Task['test:units'].invoke - #Rake::Task['test:integration'].invoke + Rake::Task['test:integration'].invoke end begin @@ -77,10 +43,8 @@ task default: :test require 'rdoc/task' Rake::RDocTask.new do |rdoc| - version = Mysql2psql::Version::STRING - rdoc.rdoc_dir = 'rdoc' - rdoc.title = "mysql2psql #{version}" - rdoc.rdoc_files.include('README*') - rdoc.rdoc_files.include('lib/**/*.rb') + rdoc.title = "Mysql2postgres #{Mysql2postgres::VERSION}" + rdoc.rdoc_files.include 'README*' + rdoc.rdoc_files.include 'lib/**/*.rb' end diff --git a/bin/mysql2postgres b/bin/mysql2postgres new file mode 100755 index 0000000..8ea1e9b --- /dev/null +++ b/bin/mysql2postgres @@ -0,0 +1,19 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +$LOAD_PATH.unshift File.join(File.dirname(__dir__), 'lib') + +require 'rubygems' +require 'bundler/setup' +require 'mysql2postgres' + +config_file = ARGV.empty? ? File.join(File.dirname(__dir__), 'config', 'database.yml') : File.expand_path(ARGV[0]) + +raise "'#{config_file}' does not exist" unless FileTest.exist? config_file + +db_yaml = YAML.safe_load_file config_file + +raise "'#{config_file}' does not contain a mysql configuration directive for conversion" unless db_yaml.key? 'mysql' +raise "'#{config_file}' does not contain destinations configuration directive for conversion" unless db_yaml.key? 'destinations' + +Mysql2postgres.new(db_yaml, config_file).convert diff --git a/bin/mysqltopostgres b/bin/mysqltopostgres deleted file mode 100755 index 0ea8322..0000000 --- a/bin/mysqltopostgres +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env ruby -$LOAD_PATH.unshift File.join(File.dirname(__FILE__), '..', 'lib') - -require 'rubygems' -require 'bundler/setup' - -require 'mysqltopostgres' - -CONFIG_FILE = File.expand_path(File.expand_path(File.dirname(__FILE__)) + '/../config/database.yml') - -if FileTest.exist?(CONFIG_FILE) || (ARGV.length > 0 && FileTest.exist?(File.expand_path(ARGV[0]))) - - if ARGV.length > 0 - file = ARGV[0] - else - file = CONFIG_FILE - end - - db_yaml = YAML.load_file file - - if db_yaml.key?('mysql2psql') - # puts db_yaml["mysql2psql"].to_s - Mysql2psql.new(db_yaml['mysql2psql']).convert - else - # Oh Noes! There is no key in the hash... - fail "'#{file}' does not contain a configuration directive for mysql -> postgres" - end - -else - fail "'#{file}' does not exist" -end diff --git a/config/default.database.yml b/config/default.database.yml index 1fcfaac..760f6e8 100644 --- a/config/default.database.yml +++ b/config/default.database.yml @@ -1,45 +1,44 @@ -default: &default - adapter: jdbcpostgresql - encoding: unicode - pool: 4 - username: terrapotamus - password: default - host: 127.0.0.1 - template: template_gis - gis_schema_name: gis_tmp - -development: &development - <<: *default - database: default_development - -test: &test - <<: *default - database: default_test - -production: &production - <<: *default - database: default_production - -mysql_data_source: &pii +mysql: hostname: localhost port: 3306 username: username password: default database: awesome_possum + encoding: utf8mb4 -mysql2psql: - mysql: - <<: *pii - - destination: - production: - <<: *production - test: - <<: *test - development: - <<: *development +# if MYSQL2POSTGRES_ENV and RAILS_ENV is missing, development is used! +destinations: + production: + # adapter: default is postgresql + adapter: postgresql + encoding: utf8 + pool: 4 + username: terrapotamus + password: default + host: 127.0.0.1 + database: awesome_possum + development: + encoding: utf8 + pool: 4 + username: terrapotamus + password: default + host: 127.0.0.1 + database: awesome_possum_dev + test: + encoding: utf8 + pool: 4 + username: terrapotamus + password: default + host: 127.0.0.1 + database: awesome_possum_test + test2: + # use adapter 'file', if you just want to create a sql file for postgresql + # NOTE: PostgreSQL import is skipped + adapter: file + # if filename is not specified, dump file name is used + filename: /tmp/converted.sql - tables: +tables: - countries - samples - universes @@ -47,23 +46,27 @@ mysql2psql: - variables - sample_variables - # If suppress_data is true, only the schema definition will be exported/migrated, and not the data - suppress_data: false +# If suppress_data is true, only the schema definition will be exported/migrated, and not the data +suppress_data: false + +# If suppress_ddl is true, only the data will be exported/imported, and not the schema +suppress_ddl: false - # If suppress_ddl is true, only the data will be exported/imported, and not the schema - suppress_ddl: true +# If force_truncate is true, forces a table truncate before table loading +force_truncate: false - # If force_truncate is true, forces a table truncate before table loading - force_truncate: false +preserve_order: true - preserve_order: true +remove_dump_file: true - remove_dump_file: true +dump_file_directory: /tmp - dump_file_directory: /tmp +# if true, migration is to dump file (PostgreSQL import is skipped) +# NOTE: do not use remove_dump_file: true, if you want to use converted sql file) +to_file: false - report_status: json # false, json, xml +report_status: json # false, json, xml - # If clear_schema is true, the public schema will be recreated before conversion - # The import will fail if both clear_schema and suppress_ddl are true. - clear_schema: false +# If clear_schema is true, the public schema will be recreated before conversion +# The import will fail if both clear_schema and suppress_ddl are true. +clear_schema: false diff --git a/lib/mysql2postgres.rb b/lib/mysql2postgres.rb new file mode 100644 index 0000000..124dffa --- /dev/null +++ b/lib/mysql2postgres.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +require 'yaml' + +require 'pg' +require 'pg_ext' +require 'pg/exceptions' +# require 'pg/constants' +require 'pg/connection' +require 'pg/result' + +require 'mysql2postgres/version' +require 'mysql2postgres/converter' +require 'mysql2postgres/mysql_reader' +require 'mysql2postgres/postgres_writer' +require 'mysql2postgres/postgres_file_writer' +require 'mysql2postgres/postgres_db_writer' + +require 'debug' if ENV.fetch('https://github.com/ENABLE_DEBUG', nil) == '1' + +class Mysql2postgres + attr_reader :options, :config_file, :reader, :writer + + def initialize(yaml, config_file = nil) + @config_file = config_file + @options = build_options yaml + end + + def convert + @reader = MysqlReader.new options + + puts "mysql2postgres #{Mysql2postgres::VERSION}" + puts "Config file: #{config_file}" + puts "Dumpfile: #{dump_file}" + + @writer = if to_file? + puts 'Target: File' + PostgresFileWriter.new dump_file, options[:destination] + else + puts "Target: PostgreSQL DB (#{adapter})" + PostgresDbWriter.new dump_file, options[:destination] + end + + Converter.new(reader, writer, options).convert + File.delete dump_file if options[:remove_dump_file] && File.exist?(dump_file) + end + + private + + def adapter + if options[:destination][:adapter].nil? || options[:destination][:adapter].empty? + 'postgresql' + else + options[:destination][:adapter] + end + end + + def environment + if ENV['MYSQL2POSTGRES_ENV'] + ENV['MYSQL2POSTGRES_ENV'] + elsif ENV['RAILS_ENV'] + ENV['RAILS_ENV'] + else + 'development' + end + end + + def to_file? + adapter == 'file' + end + + def build_options(yaml) + yaml.transform_keys(&:to_sym).tap do |opts| + opts[:mysql].transform_keys!(&:to_sym) + + destinations = opts.delete :destinations + opts[:destination] = destinations[environment]&.transform_keys(&:to_sym) + + if opts[:destination].nil? || opts[:destination].empty? + raise "no configuration for environment '#{environment}' in destinations available. Use MYSQL2POSTGRES_ENV or RAILS_ENV." + end + end + end + + def dump_file + @dump_file ||= if to_file? && options[:destination][:filename] && options[:destination][:filename] != '' + options[:destination][:filename] + else + tag = Time.new.strftime '%Y%m%d-%H%M%S' + path = options[:dump_file_directory] || './' + File.expand_path File.join(path, "output_#{tag}.sql") + end + end +end diff --git a/lib/mysql2postgres/connection.rb b/lib/mysql2postgres/connection.rb new file mode 100644 index 0000000..a2adaa6 --- /dev/null +++ b/lib/mysql2postgres/connection.rb @@ -0,0 +1,129 @@ +# frozen_string_literal: true + +class Mysql2postgres + class Connection + attr_reader :conn, + :hostname, + :login, + :password, + :database, + :schema, + :port, + :copy_manager, + :stream, + :is_copying + + def initialize(pg_options) + @hostname = pg_options[:hostname] || 'localhost' + @login = pg_options[:username] + @password = pg_options[:password] + @database = pg_options[:database] + @port = (pg_options[:port] || 5432).to_s + + @database, @schema = database.split ':' + + @conn = open + raise_nil_connection if conn.nil? + + @is_copying = false + @current_statement = '' + end + + def open + @conn = PG::Connection.open dbname: database, + user: login, + password: password, + host: hostname, + port: port + end + + # ensure that the copy is completed, in case we hadn't seen a '\.' in the data stream. + def flush + conn.put_copy_end + rescue StandardError => e + warn e + ensure + @is_copying = false + end + + def execute(sql) + if sql.match(/^COPY /) && !is_copying + # sql.chomp! # cHomp! cHomp! + conn.exec sql + @is_copying = true + elsif sql.match(/^(ALTER|CREATE|DROP|SELECT|SET|TRUNCATE) /) && !is_copying + @current_statement = sql + elsif is_copying + if sql.chomp == '\.' || sql.chomp.match(/^$/) + flush + else + begin + until conn.put_copy_data sql + warn ' waiting for connection to be writable...' + sleep 0.1 + end + rescue StandardError => e + @is_copying = false + warn e + raise e + end + end + elsif @current_statement.length.positive? + @current_statement << ' ' + @current_statement << sql + end + + return unless @current_statement.match?(/;$/) + + run_statement @current_statement + @current_statement = '' + end + + # we're done talking to the database, so close the connection cleanly. + def finish + @conn.finish + end + + # given a file containing psql syntax at path, pipe it down to the database. + def load_file(path) + if @conn + File.open path, 'r:UTF-8' do |file| + file.each_line do |line| + execute line + end + flush + end + finish + else + raise_nil_connection + end + end + + def clear_schema + statements = ['DROP SCHEMA PUBLIC CASCADE', 'CREATE SCHEMA PUBLIC'] + statements.each do |statement| + run_statement statement + end + end + + def raise_nil_connection + raise 'No Connection' + end + + def tables + result = run_statement <<~SQL_TABLES + SELECT table_name + FROM information_schema.tables + WHERE table_schema = 'public' AND table_type = 'BASE TABLE' + SQL_TABLES + + result.map { |t| t['table_name'] } + end + + private + + def run_statement(statement) + @conn.exec statement + end + end +end diff --git a/lib/mysql2postgres/converter.rb b/lib/mysql2postgres/converter.rb new file mode 100644 index 0000000..6a42e8d --- /dev/null +++ b/lib/mysql2postgres/converter.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +class Mysql2postgres + class Converter + attr_reader :reader, + :writer, + :options, + :exclude_tables, + :only_tables, + :suppress_data, + :suppress_ddl, + :force_truncate, + :preserve_order, + :clear_schema + + def initialize(reader, writer, options) + @reader = reader + @writer = writer + @exclude_tables = options[:exclude_tables] || [] + @only_tables = options[:tables] + @suppress_data = options[:suppress_data] || false + @suppress_ddl = options[:suppress_ddl] || false + @force_truncate = options[:force_truncate] || false + @preserve_order = options[:preserve_order] || false + @clear_schema = options[:clear_schema] || false + end + + def convert + tables = reader.tables + tables.reject! { |table| exclude_tables.include? table.name } + tables.select! { |table| only_tables ? only_tables.include?(table.name) : true } + + # preserve order only works, if only_tables are specified + if preserve_order && only_tables + reordered_tables = [] + + only_tables.each do |only_table| + idx = tables.index { |table| table.name == only_table } + if idx.nil? + warn "Specified source table '#{only_table}' does not exist, skiped by migration" + else + reordered_tables << tables[idx] + end + end + + tables = reordered_tables + end + + unless suppress_ddl + tables.each do |table| + puts "Writing DDL for #{table.name}" + writer.write_table table + end + end + + unless suppress_data + if force_truncate && suppress_ddl + tables.each do |table| + puts "Truncate table #{table.name}" + writer.truncate table + end + end + + tables.each do |table| + puts "Writing data for #{table.name}" + writer.write_contents table, reader + end + end + + puts 'Writing indices and constraints' + unless suppress_ddl + tables.each do |table| + writer.write_indexes table + end + end + + unless suppress_ddl + tables.each do |table| + writer.write_constraints table + end + end + + writer.close + writer.clear_schema if clear_schema + writer.inload + 0 + rescue StandardError => e + warn "mysql2postgres: Conversion failed: #{e}" + warn e + warn e.backtrace[0, 3].join("\n") + -1 + end + end +end diff --git a/lib/mysql2postgres/mysql_reader.rb b/lib/mysql2postgres/mysql_reader.rb new file mode 100644 index 0000000..704f270 --- /dev/null +++ b/lib/mysql2postgres/mysql_reader.rb @@ -0,0 +1,238 @@ +# frozen_string_literal: true + +require 'rubygems' +require 'bundler/setup' + +require 'mysql' +require 'csv' + +class Mysql2postgres + class MysqlReader + class Table + attr_reader :name + + def initialize(reader, name) + @reader = reader + @name = name + end + + def columns + @columns ||= load_columns + end + + def convert_type(type) + case type + when /int.* unsigned/, /bigint/ + 'bigint' + when 'bit(1)', 'tinyint(1)' + 'boolean' + when /tinyint/ + 'tinyint' + when /int/ + 'integer' + when /varchar/, /set/ + 'varchar' + when /char/ + 'char' + when /decimal/ + 'decimal' + when /(float|double)/ + 'double precision' + else + type + end + end + + def load_columns + @reader.reconnect + # mysql_flags = ::Mysql::Field.constants.select { |c| c.to_s.include?('FLAG') } + + fields = [] + @reader.query "EXPLAIN `#{name}`" do |res| + while (field = res.fetch_row) + length = field[1][/\((\d+)\)/, 1] if field[1].match?(/\((\d+)\)/) + length = field[1][/\((\d+),(\d+)\)/, 1] if field[1].match?(/\((\d+),(\d+)\)/) + desc = { + name: field[0], + table_name: name, + type: convert_type(field[1]), + length: length&.to_i, + decimals: field[1][/\((\d+),(\d+)\)/, 2], + null: field[2] == 'YES', + primary_key: field[3] == 'PRI', + auto_increment: field[5] == 'auto_increment' + } + desc[:default] = field[4] unless field[4].nil? + fields << desc + end + end + + fields.select { |field| field[:auto_increment] }.each do |field| + @reader.query "SELECT max(`#{field[:name]}`) FROM `#{name}`" do |res| + field[:maxval] = res.fetch_row[0].to_i + end + end + fields + end + + def indexes + load_indexes unless @indexes + @indexes + end + + def foreign_keys + load_indexes unless @foreign_keys + @foreign_keys + end + + def load_indexes + @indexes = [] + @foreign_keys = [] + + @reader.query "SHOW CREATE TABLE `#{name}`" do |result| + explain = result.fetch_row[1] + explain.split("\n").each do |line| + next unless line.include? ' KEY ' + + index = {} + if (match_data = /CONSTRAINT `(\w+)` FOREIGN KEY \((.*?)\) REFERENCES `(\w+)` \((.*?)\)(.*)/.match(line)) + index[:name] = "fk_#{name}_#{match_data[1]}" + index[:column] = match_data[2].delete!('`').split(', ') + index[:ref_table] = match_data[3] + index[:ref_column] = match_data[4].delete!('`').split(', ') + + the_rest = match_data[5] + + if (match_data = /ON DELETE (SET NULL|SET DEFAULT|RESTRICT|NO ACTION|CASCADE)/.match(the_rest)) + index[:on_delete] = match_data[1] + else + index[:on_delete] ||= 'RESTRICT' + end + + if (match_data = /ON UPDATE (SET NULL|SET DEFAULT|RESTRICT|NO ACTION|CASCADE)/.match(the_rest)) + index[:on_update] = match_data[1] + else + index[:on_update] ||= 'RESTRICT' + end + + @foreign_keys << index + elsif (match_data = /KEY `(\w+)` \((.*)\)/.match(line)) + # index[:name] = 'idx_' + name + '_' + match_data[1] + # with redmine we do not want prefix idx_tablename_ + index[:name] = match_data[1] + index[:columns] = match_data[2].split(',').map { |col| col[/`(\w+)`/, 1] } + index[:unique] = true if line.include? 'UNIQUE' + @indexes << index + elsif (match_data = /PRIMARY KEY .*\((.*)\)/.match(line)) + index[:primary] = true + index[:columns] = match_data[1].split(',').map { |col| col.strip.delete '`' } + @indexes << index + end + end + end + end + + def count_rows + @reader.query "SELECT COUNT(*) FROM `#{name}`" do |res| + return res.fetch_row[0].to_i + end + end + + def id? + !!columns.find { |col| col[:name] == 'id' } + end + + def count_for_pager + query = id? ? 'MAX(id)' : 'COUNT(*)' + @reader.query "SELECT #{query} FROM `#{name}`" do |res| + return res.fetch_row[0].to_i + end + end + + def query_for_pager + query = id? ? 'WHERE id >= ? AND id < ?' : 'LIMIT ?,?' + + cols = columns.map do |c| + if c[:type] == 'multipolygon' + "AsWKT(`#{c[:name]}`) as `#{c[:name]}`" + else + "`#{c[:name]}`" + end + end + + "SELECT #{cols.join ', '} FROM `#{name}` #{query}" + end + end + + attr_reader :mysql + + def initialize(options) + @host = options[:mysql][:hostname] + @user = options[:mysql][:username] + @passwd = options[:mysql][:password] + @db = options[:mysql][:database] + @port = if options[:mysql][:port] + options[:mysql][:port] unless options[:mysql][:port].to_s.empty? + else + 3306 + end + @sock = options[:mysql][:socket] && !options[:mysql][:socket].empty? ? options[:mysql][:socket] : nil + @flag = options[:mysql][:flag] && !options[:mysql][:flag].empty? ? options[:mysql][:flag] : nil + + connect + end + + def connect + @mysql = ::Mysql.connect @host, @user, @passwd, @db, @port, @sock + # utf8_unicode_ci :: https://rubydoc.info/gems/ruby-mysql/Mysql/Charset + @mysql.charset = ::Mysql::Charset.by_number 192 + @mysql.query 'SET NAMES utf8' + + var_info = @mysql.query "SHOW VARIABLES LIKE 'query_cache_type'" + return if var_info.nil? || var_info.first.nil? || var_info.first[1] == 'OFF' + + @mysql.query 'SET SESSION query_cache_type = OFF' + end + + def reconnect + @mysql.close + rescue StandardError + warn 'could not close previous mysql connection' + ensure + connect + end + + def query(*args, &) + mysql.query(*args, &) + rescue Mysql::Error => e + if e.message.match?(/gone away/i) + reconnect + retry + else + puts "MySQL Query failed '#{args.inspect}' #{e.inspect}" + puts e.backtrace[0, 5].join("\n") + [] + end + end + + def tables + @tables ||= @mysql.query('SHOW TABLES').map { |row| Table.new self, row.first } + end + + def paginated_read(table, page_size) + count = table.count_for_pager + return if count < 1 + + statement = @mysql.prepare table.query_for_pager + counter = 0 + 0.upto (count + page_size) / page_size do |i| + statement.execute(i * page_size, table.id? ? (i + 1) * page_size : page_size) + while (row = statement.fetch) + counter += 1 + yield row, counter + end + end + counter + end + end +end diff --git a/lib/mysql2postgres/postgres_db_writer.rb b/lib/mysql2postgres/postgres_db_writer.rb new file mode 100644 index 0000000..e5a798b --- /dev/null +++ b/lib/mysql2postgres/postgres_db_writer.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +require 'mysql2postgres/postgres_writer' +require 'mysql2postgres/connection' + +class Mysql2postgres + class PostgresDbWriter < PostgresFileWriter + attr_reader :connection + + def initialize(file, destination) + # NOTE: the superclass opens and truncates filename for writing + super + + @connection = Connection.new destination + end + + def inload(path = filename) + connection.load_file path + end + + def clear_schema + connection.clear_schema + end + end +end diff --git a/lib/mysql2postgres/postgres_file_writer.rb b/lib/mysql2postgres/postgres_file_writer.rb new file mode 100644 index 0000000..08e2089 --- /dev/null +++ b/lib/mysql2postgres/postgres_file_writer.rb @@ -0,0 +1,157 @@ +# frozen_string_literal: true + +require 'mysql2postgres/postgres_writer' +require 'fileutils' + +class Mysql2postgres + class PostgresFileWriter < PostgresWriter + def initialize(file, destination) + super() + + @filename = file + @destination = destination + + @f = File.open file, 'w+:UTF-8' + @f << <<~SQL_HEADER + -- MySQL 2 PostgreSQL dump\n + SET client_encoding = 'UTF8'; + SET standard_conforming_strings = off; + SET check_function_bodies = false; + SET client_min_messages = warning; + + SQL_HEADER + end + + def truncate(table) + serial_key = nil + maxval = nil + + table.columns.map do |column| + if column[:auto_increment] + serial_key = column[:name] + maxval = column[:maxval].to_i < 1 ? 1 : column[:maxval] + 1 + end + end + + @f << <<~SQL_TRUNCATE + -- TRUNCATE #{table.name}; + TRUNCATE #{PG::Connection.quote_ident table.name} CASCADE; + + SQL_TRUNCATE + + return unless serial_key + + @f << <<~SQL_SERIAL + SELECT pg_catalog.setval(pg_get_serial_sequence('#{table.name}', '#{serial_key}'), #{maxval}, true); + SQL_SERIAL + end + + def write_table(table) + primary_keys = [] + serial_key = nil + maxval = nil + + columns = table.columns.map do |column| + if column[:auto_increment] + serial_key = column[:name] + maxval = column[:maxval].to_i < 1 ? 1 : column[:maxval] + 1 + end + primary_keys << column[:name] if column[:primary_key] + " #{column_description column}" + end.join(",\n") + + if serial_key + @f << <<~SQL_SEQUENCE + -- + -- Name: #{table.name}_#{serial_key}_seq; Type: SEQUENCE; Schema: public + -- + + DROP SEQUENCE IF EXISTS #{table.name}_#{serial_key}_seq CASCADE; + + CREATE SEQUENCE #{table.name}_#{serial_key}_seq + INCREMENT BY 1 + NO MAXVALUE + NO MINVALUE + CACHE 1; + + + SELECT pg_catalog.setval('#{table.name}_#{serial_key}_seq', #{maxval}, true); + + SQL_SEQUENCE + end + + @f << <<~SQL_TABLE + -- Table: #{table.name} + + -- DROP TABLE #{table.name}; + DROP TABLE IF EXISTS #{PG::Connection.quote_ident table.name} CASCADE; + + CREATE TABLE #{PG::Connection.quote_ident table.name} ( + SQL_TABLE + + @f << columns + + if (primary_index = table.indexes.find { |index| index[:primary] }) + @f << ",\n CONSTRAINT #{table.name}_pkey PRIMARY KEY(#{quoted_list primary_index[:columns]})" + end + + @f << <<~SQL_OIDS + \n) + WITHOUT OIDS; + SQL_OIDS + + table.indexes.each do |index| + next if index[:primary] + + unique = index[:unique] ? 'UNIQUE ' : nil + @f << <<~SQL_INDEX + DROP INDEX IF EXISTS #{PG::Connection.quote_ident index[:name]} CASCADE; + CREATE #{unique}INDEX #{PG::Connection.quote_ident index[:name]} + ON #{PG::Connection.quote_ident table.name} (#{quoted_list index[:columns]}); + SQL_INDEX + end + end + + def write_indexes(_table); end + + def write_constraints(table) + table.foreign_keys.each do |key| + @f << "ALTER TABLE #{PG::Connection.quote_ident table.name} " \ + "ADD FOREIGN KEY (#{quoted_list key[:column]}) " \ + "REFERENCES #{PG::Connection.quote_ident key[:ref_table]}(#{quoted_list key[:ref_column]}) " \ + "ON UPDATE #{key[:on_update]} ON DELETE #{key[:on_delete]};\n" + end + end + + def write_contents(table, reader) + @f << <<~SQL_COPY + -- + -- Data for Name: #{table.name}; Type: TABLE DATA; Schema: public + -- + + COPY "#{table.name}" (#{quoted_list(table.columns.map { |m| m[:name] })}) FROM stdin; + SQL_COPY + + reader.paginated_read table, 1000 do |row, _counter| + process_row table, row + @f << row.join("\t") + @f << "\n" + end + @f << "\\.\n\n" + end + + def close + @f.close + end + + def inload + puts "\nSkip import to PostgreSQL DB. SQL file created successfully." + end + + private + + def quoted_list(list) + list.map { |c| PG::Connection.quote_ident c }.join(', ') + end + end +end diff --git a/lib/mysql2postgres/postgres_writer.rb b/lib/mysql2postgres/postgres_writer.rb new file mode 100644 index 0000000..50bf8bd --- /dev/null +++ b/lib/mysql2postgres/postgres_writer.rb @@ -0,0 +1,168 @@ +# frozen_string_literal: true + +require 'zlib' + +class Mysql2postgres + class PostgresWriter + attr_reader :filename, :destination + + def column_description(column) + "#{PG::Connection.quote_ident column[:name]} #{column_type_info column}" + end + + def column_type(column) + column_type_info(column).split.first + end + + def column_type_info(column) + return "integer DEFAULT nextval('#{column[:table_name]}_#{column[:name]}_seq'::regclass) NOT NULL" if column[:auto_increment] + + default = if column[:default] + " DEFAULT #{column[:default].nil? ? 'NULL' : "'#{PG::Connection.escape column[:default]}'"}" + end + null = column[:null] ? '' : ' NOT NULL' + type = case column[:type] + # String types + when 'char' + default += '::char' if default + "character(#{column[:length]})" + when 'varchar' + default += '::character varying' if default + "character varying(#{column[:length]})" + # Integer and numeric types + when 'integer' + default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default].to_i}" if default + 'integer' + when 'bigint' + default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default].to_i}" if default + 'bigint' + when 'tinyint' + default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default].to_i}" if default + 'smallint' + when 'boolean' + default = " DEFAULT #{column[:default].to_i == 1 ? 'true' : 'false'}" if default + 'boolean' + when 'float', 'float unsigned' + default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default].to_f}" if default + 'real' + when 'decimal' + default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default]}" if default + "numeric(#{column[:length] || 10}, #{column[:decimals] || 0})" + when 'double precision' + default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default]}" if default + 'double precision' + when 'datetime', 'datetime(6)' + default = nil + 'timestamp without time zone' + when 'date' + default = nil + 'date' + when 'timestamp' + case column[:default] + when 'CURRENT_TIMESTAMP' + default = ' DEFAULT CURRENT_TIMESTAMP' + when datetime_zero + default = " DEFAULT '#{datetime_zero_fix}'" + when datetime_zero(with_seconds: true) + default = " DEFAULT '#{datetime_zero_fix with_seconds: true}'" + end + 'timestamp without time zone' + when 'time' + default = ' DEFAULT NOW()' if default + 'time without time zone' + when 'blob', 'longblob', 'mediumblob', 'tinyblob', 'varbinary' + 'bytea' + when 'text', 'tinytext', 'mediumtext', 'longtext' + 'text' + when /^enum/ + default += '::character varying' if default + enum = column[:type].gsub(/enum|\(|\)/, '') + max_enum_size = enum.split(',').map { |check| check.size - 2 }.max + "character varying(#{max_enum_size}) check( \"#{column[:name]}\" in (#{enum}))" + when 'geometry', 'multipolygon' + 'geometry' + else + puts "Unknown #{column.inspect}" + column[:type].inspect + return '' + end + + "#{type}#{default}#{null}" + end + + def process_row(table, row) + table.columns.each_with_index do |column, index| + row[index] = Time.at(row[index]).utc.strftime('%H:%M:%S') if column[:type] == 'time' && row[index] + + if row[index].is_a? Time + row[index] = row[index].to_s.gsub datetime_zero, datetime_zero_fix + row[index] = row[index].to_s.gsub datetime_zero(with_seconds: true), datetime_zero_fix(with_seconds: true) + end + + if column_type(column) == 'boolean' + row[index] = if row[index] == 1 + 't' + elsif row[index]&.zero? + 'f' + else + row[index] + end + end + + row[index] = string_data table, row, index, column if row[index].is_a? String + + row[index] = '\N' unless row[index] + end + end + + def truncate(_table) end + + def inload + raise "Method 'inload' needs to be overridden..." + end + + private + + def datetime_zero(with_seconds: false) + datetime_value date: '0000-00-00', with_seconds: with_seconds + end + + def datetime_zero_fix(with_seconds: false) + datetime_value date: '1970-01-01', with_seconds: with_seconds + end + + def datetime_value(date:, with_seconds: false) + value = ["#{date} 00:00"] + value << '00' if with_seconds + value.join ':' + end + + def string_data(table, row, index, column) + if column_type(column) == 'bytea' + if column[:name] == 'data' + with_gzip = false + table.columns.each_with_index do |column_data, index_data| + if column_data[:name] == 'compression' && row[index_data] == 'gzip' + with_gzip = true + break + end + end + + escape_bytea(with_gzip ? Zlib::Inflate.inflate(row[index]) : row[index]) + else + escape_bytea row[index] + end + else + escape_data(row[index]).gsub("\n", '\n').gsub("\t", '\t').gsub("\r", '\r').gsub(/\0/, '') + end + end + + def escape_bytea(data) + escape_data(PG::Connection.escape_bytea(data)).gsub("''", "'") + end + + def escape_data(value) + value.gsub '\\', '\\\\\\' + end + end +end diff --git a/lib/mysql2postgres/version.rb b/lib/mysql2postgres/version.rb new file mode 100644 index 0000000..a75e37a --- /dev/null +++ b/lib/mysql2postgres/version.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +class Mysql2postgres + VERSION = '0.4.3' +end diff --git a/lib/mysql2psql/config.rb b/lib/mysql2psql/config.rb deleted file mode 100644 index 7da0a55..0000000 --- a/lib/mysql2psql/config.rb +++ /dev/null @@ -1,9 +0,0 @@ -require 'mysql2psql/config_base' - -class Mysql2psql - class Config < ConfigBase - def initialize(yaml) - super(yaml) - end - end -end diff --git a/lib/mysql2psql/config_base.rb b/lib/mysql2psql/config_base.rb deleted file mode 100644 index 6397e57..0000000 --- a/lib/mysql2psql/config_base.rb +++ /dev/null @@ -1,35 +0,0 @@ -require 'yaml' -require 'mysql2psql/errors' - -class Mysql2psql - class ConfigBase - attr_reader :config - - def initialize(yaml) - @config = yaml - end - - def [](key) - send(key) - end - - def method_missing(name, *args) - token = name.to_s - default = args.length > 0 ? args[0] : '' - must_be_defined = default == :none - case token - when /mysql/i - key = token.sub(/^mysql/, '') - value = config['mysql'][key] - when /dest/i - key = token.sub(/^dest/, '') - value = config['destination'][key] - when /only_tables/i - value = config['tables'] - else - value = config[token] - end - value.nil? ? (must_be_defined ? (fail Mysql2psql::UninitializedValueError.new("no value and no default for #{name}")) : default) : value - end - end -end diff --git a/lib/mysql2psql/connection.rb b/lib/mysql2psql/connection.rb deleted file mode 100644 index e458aa3..0000000 --- a/lib/mysql2psql/connection.rb +++ /dev/null @@ -1,183 +0,0 @@ -class Mysql2psql - class Connection - attr_reader :conn, :adapter, :hostname, :login, :password, :database, :schema, :port, :environment, :jruby, :copy_manager, :stream, :is_copying - - def initialize(options) - # Rails-centric stuffs - - @environment = ENV['RAILS_ENV'].nil? ? 'development' : ENV['RAILS_ENV'] - - if options.key?('config') && options['config'].key?('destination') && options['config']['destination'].key?(environment) - - pg_options = Config.new(YAML.load(options['config']['destination'][environment].to_yaml)) - @hostname, @login, @password, @database, @port = pg_options.host('localhost'), pg_options.username, pg_options.password, pg_options.database, pg_options.port(5432).to_s - @database, @schema = database.split(':') - - @adapter = pg_options.adapter('jdbcpostgresql') - - else - fail 'Unable to locate PostgreSQL destination environment in the configuration file' - end - - if RUBY_PLATFORM == 'java' - @jruby = true - - ActiveRecord::Base.establish_connection( - adapter: adapter, - database: database, - username: login, - password: password, - host: hostname, - port: port) - - @conn = ActiveRecord::Base.connection_pool.checkout - - unless conn.nil? - raw_connection = conn.raw_connection - @copy_manager = org.postgresql.copy.CopyManager.new(raw_connection.connection) - else - raise_nil_connection - end - - else - @jruby = false - - @conn = PG::Connection.open(dbname: database, user: login, password: password, host: hostname, port: port) - - if conn.nil? - raise_nil_connection - end - - end - - @is_copying = false - @current_statement = '' - end - - # ensure that the copy is completed, in case we hadn't seen a '\.' in the data stream. - def flush - if jruby - stream.end_copy if @is_copying - else - conn.put_copy_end - end - rescue => e - $stderr.puts e - ensure - @is_copying = false - end - - def execute(sql) - if sql.match(/^COPY /) && !is_copying - # sql.chomp! # cHomp! cHomp! - - if jruby - @stream = copy_manager.copy_in(sql) - else - conn.exec(sql) - end - - @is_copying = true - - elsif sql.match(/^(ALTER|CREATE|DROP|SELECT|SET|TRUNCATE) /) && !is_copying - - @current_statement = sql - - else - - if is_copying - - if sql.chomp == '\.' || sql.chomp.match(/^$/) - - flush - - else - - if jruby - - begin - row = sql.to_java_bytes - stream.write_to_copy(row, 0, row.length) - - rescue => e - - stream.cancel_copy - @is_copying = false - $stderr.puts e - - raise e - end - - else - - begin - - until conn.put_copy_data(sql) - $stderr.puts ' waiting for connection to be writable...' - sleep 0.1 - end - - rescue => e - @is_copying = false - $stderr.puts e - raise e - end - end - end - elsif @current_statement.length > 0 - @current_statement << ' ' - @current_statement << sql - else - # maybe a comment line? - end - end - - if @current_statement.match(/;$/) - run_statement(@current_statement) - @current_statement = '' - end - end - - # we're done talking to the database, so close the connection cleanly. - def finish - if jruby - ActiveRecord::Base.connection_pool.checkin(@conn) if @conn - else - @conn.finish if @conn - end - end - - # given a file containing psql syntax at path, pipe it down to the database. - def load_file(path) - if @conn - File.open(path, 'r:UTF-8') do |file| - file.each_line do |line| - execute(line) - end - flush - end - finish - else - raise_nil_connection - end - end - - def clear_schema - statements = ['DROP SCHEMA PUBLIC CASCADE', 'CREATE SCHEMA PUBLIC'] - statements.each do |statement| - run_statement(statement) - end - end - - def raise_nil_connection - fail 'No Connection' - end - - private - - def run_statement(statement) - method = jruby ? :execute : :exec - @conn.send(method, statement) - end - end -end diff --git a/lib/mysql2psql/converter.rb b/lib/mysql2psql/converter.rb deleted file mode 100644 index 4f9312b..0000000 --- a/lib/mysql2psql/converter.rb +++ /dev/null @@ -1,81 +0,0 @@ -class Mysql2psql - class Converter - attr_reader :reader, :writer, :options - attr_reader :exclude_tables, :only_tables, :suppress_data, :suppress_ddl, :force_truncate, :preserve_order, :clear_schema - - def initialize(reader, writer, options) - @reader = reader - @writer = writer - @options = options - @exclude_tables = options.exclude_tables([]) - @only_tables = options.only_tables(nil) - @suppress_data = options.suppress_data(false) - @suppress_ddl = options.suppress_ddl(false) - @force_truncate = options.force_truncate(false) - @preserve_order = options.preserve_order(false) - @clear_schema = options.clear_schema(false) - end - - def convert - tables = reader.tables - .reject { |table| @exclude_tables.include?(table.name) } - .select { |table| @only_tables ? @only_tables.include?(table.name) : true } - - if @preserve_order - - reordered_tables = [] - - @only_tables.each do |only_table| - idx = tables.index { |table| table.name == only_table } - reordered_tables << tables[idx] - end - - tables = reordered_tables - - end - - tables.each do |table| - writer.write_table(table) - end unless @suppress_ddl - - # tables.each do |table| - # writer.truncate(table) if force_truncate && suppress_ddl - # writer.write_contents(table, reader) - # end unless @suppress_data - - unless @suppress_data - - tables.each do |table| - writer.truncate(table) if force_truncate && suppress_ddl - end - - tables.each do |table| - writer.write_contents(table, reader) - end - - end - - tables.each do |table| - writer.write_indexes(table) - end unless @suppress_ddl - tables.each do |table| - writer.write_constraints(table) - end unless @suppress_ddl - - writer.close - - if @clear_schema - writer.clear_schema - end - - writer.inload - - 0 - rescue => e - $stderr.puts "Mysql2psql: Conversion failed: #{e.to_s}" - $stderr.puts e - $stderr.puts e.backtrace[0,3].join("\n") - return -1 - end - end -end diff --git a/lib/mysql2psql/errors.rb b/lib/mysql2psql/errors.rb deleted file mode 100644 index 0c4b969..0000000 --- a/lib/mysql2psql/errors.rb +++ /dev/null @@ -1,16 +0,0 @@ -class Mysql2psql - class GeneralError < StandardError - end - - class ConfigurationError < StandardError - end - - class UninitializedValueError < ConfigurationError - end - - class ConfigurationFileNotFound < ConfigurationError - end - - class ConfigurationFileInitialized < ConfigurationError - end -end diff --git a/lib/mysql2psql/mysql_reader.rb b/lib/mysql2psql/mysql_reader.rb deleted file mode 100644 index b6aa038..0000000 --- a/lib/mysql2psql/mysql_reader.rb +++ /dev/null @@ -1,233 +0,0 @@ -require 'rubygems' -require 'bundler/setup' - -require 'mysql-pr' -require 'csv' - -class Mysql2psql - class MysqlReader - class Field - end - - class Table - attr_reader :name - - def initialize(reader, name) - @reader = reader - @name = name - end - - @@types = %w(tiny enum decimal short long float double null timestamp longlong int24 date time datetime year set blob string var_string char).reduce({}) do |list, type| - list[eval("::MysqlPR::Field::TYPE_#{type.upcase}")] = type - list - end - @@types[246] = 'decimal' - - def columns - @columns ||= load_columns - end - - def convert_type(type) - case type - when /int.* unsigned/ - 'bigint' - when /bigint/ - 'bigint' - when 'bit(1)' - 'boolean' - when 'tinyint(1)' - 'boolean' - when /tinyint/ - 'tinyint' - when /int/ - 'integer' - when /varchar/ - 'varchar' - when /char/ - 'char' - when /decimal/ - 'decimal' - when /(float|double)/ - 'double precision' - when /set/ - 'varchar' - else - type - end - end - - def load_columns - @reader.reconnect - result = @reader.mysql.list_fields(name) - mysql_flags = ::MysqlPR::Field.constants.select { |c| c =~ /FLAG/ } - fields = [] - @reader.query("EXPLAIN `#{name}`") do |res| - while field = res.fetch_row - length = field[1][/\((\d+)\)/, 1] if field[1] =~ /\((\d+)\)/ - length = field[1][/\((\d+),(\d+)\)/, 1] if field[1] =~ /\((\d+),(\d+)\)/ - desc = { - name: field[0], - table_name: name, - type: convert_type(field[1]), - length: length && length.to_i, - decimals: field[1][/\((\d+),(\d+)\)/, 2], - null: field[2] == 'YES', - primary_key: field[3] == 'PRI', - auto_increment: field[5] == 'auto_increment' - } - desc[:default] = field[4] unless field[4].nil? - fields << desc - end - end - - fields.select { |field| field[:auto_increment] }.each do |field| - @reader.query("SELECT max(`#{field[:name]}`) FROM `#{name}`") do |res| - field[:maxval] = res.fetch_row[0].to_i - end - end - fields - end - def indexes - load_indexes unless @indexes - @indexes - end - - def foreign_keys - load_indexes unless @foreign_keys - @foreign_keys - end - - def load_indexes - @indexes = [] - @foreign_keys = [] - - @reader.query("SHOW CREATE TABLE `#{name}`") do |result| - explain = result.fetch_row[1] - explain.split(/\n/).each do |line| - next unless line =~ / KEY / - index = {} - if match_data = /CONSTRAINT `(\w+)` FOREIGN KEY \((.*?)\) REFERENCES `(\w+)` \((.*?)\)(.*)/.match(line) - index[:name] = 'fk_' + name + '_' + match_data[1] - index[:column] = match_data[2].gsub!('`', '').split(', ') - index[:ref_table] = match_data[3] - index[:ref_column] = match_data[4].gsub!('`', '').split(', ') - - the_rest = match_data[5] - - if match_data = /ON DELETE (SET NULL|SET DEFAULT|RESTRICT|NO ACTION|CASCADE)/.match(the_rest) - index[:on_delete] = match_data[1] - else - index[:on_delete] ||= 'RESTRICT' - end - - if match_data = /ON UPDATE (SET NULL|SET DEFAULT|RESTRICT|NO ACTION|CASCADE)/.match(the_rest) - index[:on_update] = match_data[1] - else - index[:on_update] ||= 'RESTRICT' - end - - @foreign_keys << index - elsif match_data = /KEY `(\w+)` \((.*)\)/.match(line) - index[:name] = 'idx_' + name + '_' + match_data[1] - index[:columns] = match_data[2].split(',').map { |col| col[/`(\w+)`/, 1] } - index[:unique] = true if line =~ /UNIQUE/ - @indexes << index - elsif match_data = /PRIMARY KEY .*\((.*)\)/.match(line) - index[:primary] = true - index[:columns] = match_data[1].split(',').map { |col| col.strip.gsub(/`/, '') } - @indexes << index - end - end - end - end - - def count_rows - @reader.query("SELECT COUNT(*) FROM `#{name}`") do |res| - return res.fetch_row[0].to_i - end - end - - def has_id? - !!columns.find { |col| col[:name] == 'id' } - end - - def count_for_pager - query = has_id? ? 'MAX(id)' : 'COUNT(*)' - @reader.query("SELECT #{query} FROM `#{name}`") do |res| - return res.fetch_row[0].to_i - end - end - - def query_for_pager - query = has_id? ? 'WHERE id >= ? AND id < ?' : 'LIMIT ?,?' - - cols = columns.map do |c| - if "multipolygon" == c[:type] - "AsWKT(`#{c[:name]}`) as `#{c[:name]}`" - else - "`#{c[:name]}`" - end - end - - "SELECT #{cols.join(", ")} FROM `#{name}` #{query}" - end - end - - def connect - @mysql = ::MysqlPR.connect(@host, @user, @passwd, @db, @port, @sock) - @mysql.charset = ::MysqlPR::Charset.by_number 192 # utf8_unicode_ci :: http://rubydoc.info/gems/mysql-pr/MysqlPR/Charset - @mysql.query('SET NAMES utf8') - @mysql.query('SET SESSION query_cache_type = OFF') - end - - def reconnect - @mysql.close rescue false - connect - end - - def query(*args, &block) - self.mysql.query(*args, &block) - rescue Mysql::Error => e - if e.message =~ /gone away/i - self.reconnect - retry - else - puts "MySQL Query failed '#{args.inspect}' #{e.inspect}" - puts e.backtrace[0,5].join("\n") - return [] - end - end - - def initialize(options) - @host, @user, @passwd, @db, @port, @sock, @flag = - options.mysqlhost('localhost'), options.mysqlusername, - options.mysqlpassword, options.mysqldatabase, - options.mysqlport(3306), options.mysqlsocket, - options.mysqlflag - @sock = nil if @sock == '' - @flag = nil if @flag == '' - connect - end - - attr_reader :mysql - - def tables - @tables ||= @mysql.list_tables.map { |table| Table.new(self, table) } - end - - def paginated_read(table, page_size) - count = table.count_for_pager - return if count < 1 - statement = @mysql.prepare(table.query_for_pager) - counter = 0 - 0.upto((count + page_size) / page_size) do |i| - statement.execute(i * page_size, table.has_id? ? (i + 1) * page_size : page_size) - while row = statement.fetch - counter += 1 - yield(row, counter) - end - end - counter - end - end -end \ No newline at end of file diff --git a/lib/mysql2psql/postgres_db_writer.rb b/lib/mysql2psql/postgres_db_writer.rb deleted file mode 100644 index 6476d4e..0000000 --- a/lib/mysql2psql/postgres_db_writer.rb +++ /dev/null @@ -1,23 +0,0 @@ -require 'mysql2psql/postgres_writer' -require 'mysql2psql/connection' - -class Mysql2psql - class PostgresDbWriter < PostgresFileWriter - attr_reader :connection, :filename - - def initialize(filename, options) - # note that the superclass opens and truncates filename for writing - super(filename) - @filename = filename - @connection = Connection.new(options) - end - - def inload(path = filename) - connection.load_file(path) - end - - def clear_schema - connection.clear_schema - end - end -end \ No newline at end of file diff --git a/lib/mysql2psql/postgres_file_writer.rb b/lib/mysql2psql/postgres_file_writer.rb deleted file mode 100644 index 4526ab1..0000000 --- a/lib/mysql2psql/postgres_file_writer.rb +++ /dev/null @@ -1,138 +0,0 @@ -require 'mysql2psql/postgres_writer' - -class Mysql2psql - class PostgresFileWriter < PostgresWriter - def initialize(file) - @f = File.open(file, 'w+:UTF-8') - @f << <<-EOF --- MySQL 2 PostgreSQL dump\n -SET client_encoding = 'UTF8'; -SET standard_conforming_strings = off; -SET check_function_bodies = false; -SET client_min_messages = warning; - -EOF - end - - def truncate(table) - serial_key = nil - maxval = nil - - table.columns.map do |column| - if column[:auto_increment] - serial_key = column[:name] - maxval = column[:maxval].to_i < 1 ? 1 : column[:maxval] + 1 - end - end - - @f << <<-EOF --- TRUNCATE #{table.name}; -TRUNCATE #{PGconn.quote_ident(table.name)} CASCADE; - -EOF - if serial_key - @f << <<-EOF -SELECT pg_catalog.setval(pg_get_serial_sequence('#{table.name}', '#{serial_key}'), #{maxval}, true); -EOF - end - end - - def write_table(table) - primary_keys = [] - serial_key = nil - maxval = nil - - columns = table.columns.map do |column| - if column[:auto_increment] - serial_key = column[:name] - maxval = column[:maxval].to_i < 1 ? 1 : column[:maxval] + 1 - end - if column[:primary_key] - primary_keys << column[:name] - end - ' ' + column_description(column) - end.join(",\n") - - if serial_key - - @f << <<-EOF --- --- Name: #{table.name}_#{serial_key}_seq; Type: SEQUENCE; Schema: public --- - -DROP SEQUENCE IF EXISTS #{table.name}_#{serial_key}_seq CASCADE; - -CREATE SEQUENCE #{table.name}_#{serial_key}_seq - INCREMENT BY 1 - NO MAXVALUE - NO MINVALUE - CACHE 1; - - -SELECT pg_catalog.setval('#{table.name}_#{serial_key}_seq', #{maxval}, true); - - EOF - end - - @f << <<-EOF --- Table: #{table.name} - --- DROP TABLE #{table.name}; -DROP TABLE IF EXISTS #{PGconn.quote_ident(table.name)} CASCADE; - -CREATE TABLE #{PGconn.quote_ident(table.name)} ( -EOF - - @f << columns - - if primary_index = table.indexes.find { |index| index[:primary] } - @f << ",\n CONSTRAINT #{table.name}_pkey PRIMARY KEY(#{primary_index[:columns].map { |col| PGconn.quote_ident(col) }.join(', ')})" - end - - @f << <<-EOF -\n) -WITHOUT OIDS; -EOF - - table.indexes.each do |index| - next if index[:primary] - unique = index[:unique] ? 'UNIQUE ' : nil - @f << <<-EOF -DROP INDEX IF EXISTS #{PGconn.quote_ident(index[:name])} CASCADE; -CREATE #{unique}INDEX #{PGconn.quote_ident(index[:name])} ON #{PGconn.quote_ident(table.name)} (#{index[:columns].map { |col| PGconn.quote_ident(col) }.join(', ')}); -EOF - end - end - - def write_indexes(_table) - end - - def write_constraints(table) - table.foreign_keys.each do |key| - @f << "ALTER TABLE #{PGconn.quote_ident(table.name)} ADD FOREIGN KEY (#{key[:column].map { |c|PGconn.quote_ident(c) }.join(', ')}) REFERENCES #{PGconn.quote_ident(key[:ref_table])}(#{key[:ref_column].map { |c|PGconn.quote_ident(c) }.join(', ')}) ON UPDATE #{key[:on_update]} ON DELETE #{key[:on_delete]};\n" - end - end - - def write_contents(table, reader) - @f << <<-EOF --- --- Data for Name: #{table.name}; Type: TABLE DATA; Schema: public --- - -COPY "#{table.name}" (#{table.columns.map { |column| PGconn.quote_ident(column[:name]) }.join(', ')}) FROM stdin; -EOF - - reader.paginated_read(table, 1000) do |row, _counter| - line = [] - process_row(table, row) - @f << row.join("\t") + "\n" - end - @f << "\\.\n\n" - # @f << "VACUUM FULL ANALYZE #{PGconn.quote_ident(table.name)};\n\n" - end - - def close - @f.close - end - end -end diff --git a/lib/mysql2psql/postgres_writer.rb b/lib/mysql2psql/postgres_writer.rb deleted file mode 100644 index ffc9a45..0000000 --- a/lib/mysql2psql/postgres_writer.rb +++ /dev/null @@ -1,142 +0,0 @@ -require 'mysql2psql/writer' - -class Mysql2psql - class PostgresWriter < Writer - def column_description(column) - "#{PGconn.quote_ident(column[:name])} #{column_type_info(column)}" - end - - def column_type(column) - column_type_info(column).split(' ').first - end - - def column_type_info(column) - if column[:auto_increment] - return "integer DEFAULT nextval('#{column[:table_name]}_#{column[:name]}_seq'::regclass) NOT NULL" - end - - default = column[:default] ? " DEFAULT #{column[:default].nil? ? 'NULL' : "'" + PGconn.escape(column[:default]) + "'"}" : nil - null = column[:null] ? '' : ' NOT NULL' - type = - case column[:type] - - # String types - when 'char' - default = default + '::char' if default - "character(#{column[:length]})" - when 'varchar' - default = default + '::character varying' if default - # puts "VARCHAR: #{column.inspect}" - "character varying(#{column[:length]})" - - # Integer and numeric types - when 'integer' - default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default].to_i}" if default - 'integer' - when 'bigint' - default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default].to_i}" if default - 'bigint' - when 'tinyint' - default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default].to_i}" if default - 'smallint' - - when 'boolean' - default = " DEFAULT #{column[:default].to_i == 1 ? 'true' : 'false'}" if default - 'boolean' - when 'float' - default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default].to_f}" if default - 'real' - when 'float unsigned' - default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default].to_f}" if default - 'real' - when 'decimal' - default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default]}" if default - "numeric(#{column[:length] || 10}, #{column[:decimals] || 0})" - - when 'double precision' - default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default]}" if default - 'double precision' - - # Mysql datetime fields - when 'datetime' - default = nil - 'timestamp without time zone' - when 'date' - default = nil - 'date' - when 'timestamp' - default = ' DEFAULT CURRENT_TIMESTAMP' if column[:default] == 'CURRENT_TIMESTAMP' - default = " DEFAULT '1970-01-01 00:00'" if column[:default] == '0000-00-00 00:00' - default = " DEFAULT '1970-01-01 00:00:00'" if column[:default] == '0000-00-00 00:00:00' - 'timestamp without time zone' - when 'time' - default = ' DEFAULT NOW()' if default - 'time without time zone' - - when 'tinyblob' - 'bytea' - when 'mediumblob' - 'bytea' - when 'longblob' - 'bytea' - when 'blob' - 'bytea' - when 'varbinary' - 'bytea' - when 'tinytext' - 'text' - when 'mediumtext' - 'text' - when 'longtext' - 'text' - when 'text' - 'text' - when /^enum/ - default = default + '::character varying' if default - enum = column[:type].gsub(/enum|\(|\)/, '') - max_enum_size = enum.split(',').map { |check| check.size - 2 }.sort[-1] - "character varying(#{max_enum_size}) check( \"#{column[:name]}\" in (#{enum}))" - when 'multipolygon' - 'geometry' - when 'geometry' - 'geometry' - else - puts "Unknown #{column.inspect}" - column[:type].inspect - return '' - end - "#{type}#{default}#{null}" - end - - def process_row(table, row) - table.columns.each_with_index do |column, index| - - if column[:type] == 'time' - row[index] = '%02d:%02d:%02d' % [row[index].hour, row[index].minute, row[index].second] unless row[index].nil? - end - - if row[index].is_a?(MysqlPR::Time) - row[index] = row[index].to_s.gsub('0000-00-00 00:00', '1970-01-01 00:00') - row[index] = row[index].to_s.gsub('0000-00-00 00:00:00', '1970-01-01 00:00:00') - end - - if column_type(column) == 'boolean' - row[index] = row[index] == 1 ? 't' : row[index] == 0 ? 'f' : row[index] - end - - if row[index].is_a?(String) - if column_type(column) == 'bytea' - row[index] = PGconn.escape_bytea(row[index]) - else - row[index] = row[index].gsub(/\\/, '\\\\\\').gsub(/\n/, '\n').gsub(/\t/, '\t').gsub(/\r/, '\r').gsub(/\0/, '') - end - end - - row[index] = '\N' unless row[index] - end - end - - def truncate(_table) - end - end -end diff --git a/lib/mysql2psql/version.rb b/lib/mysql2psql/version.rb deleted file mode 100644 index db7898c..0000000 --- a/lib/mysql2psql/version.rb +++ /dev/null @@ -1,9 +0,0 @@ -class Mysql2psql - module Version - MAJOR = 0 - MINOR = 3 - PATCH = 1 - - STRING = [MAJOR, MINOR, PATCH].compact.join('.') - end -end diff --git a/lib/mysql2psql/writer.rb b/lib/mysql2psql/writer.rb deleted file mode 100644 index 5389139..0000000 --- a/lib/mysql2psql/writer.rb +++ /dev/null @@ -1,7 +0,0 @@ -class Mysql2psql - class Writer - def inload - fail "Method 'inload' needs to be overridden..." - end - end -end diff --git a/lib/mysqltopostgres.rb b/lib/mysqltopostgres.rb deleted file mode 100644 index f261ecf..0000000 --- a/lib/mysqltopostgres.rb +++ /dev/null @@ -1,56 +0,0 @@ -if RUBY_PLATFORM == 'java' - require 'active_record' - require 'postgres-pr/postgres-compat' -else - require 'pg' - require 'pg_ext' - require 'pg/exceptions' - require 'pg/constants' - require 'pg/connection' - require 'pg/result' -end - -require 'mysql2psql/errors' -require 'mysql2psql/version' -require 'mysql2psql/config' -require 'mysql2psql/converter' -require 'mysql2psql/mysql_reader' -require 'mysql2psql/writer' -require 'mysql2psql/postgres_writer' -require 'mysql2psql/postgres_file_writer.rb' -require 'mysql2psql/postgres_db_writer.rb' - -class Mysql2psql - attr_reader :options, :reader, :writer - - def initialize(yaml) - @options = Config.new(yaml) - end - - def send_file_to_postgres(path) - connection = Connection.new(options) - connection.load_file(path) - end - - def convert - @reader = MysqlReader.new(options) - - tag = Time.new.to_s.gsub(/((\-)|( )|(:))+/, '') - - path = './' - - unless options.config['dump_file_directory'].nil? - path = options.config['dump_file_directory'] - end - - filename = File.expand_path(File.join(path, tag + '_output.sql')) - - @writer = PostgresDbWriter.new(filename, options) - - Converter.new(reader, writer, options).convert - - if options.config['remove_dump_file'] - File.delete filename if File.exist?(filename) - end - end -end diff --git a/mysql2postgres.gemspec b/mysql2postgres.gemspec new file mode 100644 index 0000000..814ab7e --- /dev/null +++ b/mysql2postgres.gemspec @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +lib = File.expand_path '../lib', __FILE__ +$LOAD_PATH.unshift lib unless $LOAD_PATH.include? lib +require 'mysql2postgres/version' + +Gem::Specification.new do |s| + s.name = 'mysql2postgres' + s.version = Mysql2postgres::VERSION + s.licenses = ['MIT'] + + s.authors = [ + 'Max Lapshin ', + 'Anton Ageev ', + 'Samuel Tribehou ', + 'Marco Nenciarini ', + 'James Nobis ', + 'quel ', + 'Holger Amann ', + 'Maxim Dobriakov ', + 'Michael Kimsal ', + 'Jacob Coby ', + 'Neszt Tibor ', + 'Miroslav Kratochvil ', + 'Paul Gallagher ', + 'Alex C Jokela ', + 'Peter Clark ', + 'Juga Paazmaya ', + 'Alexander Meindl 'true' } + s.executables = ['mysql2postgres'] + s.required_ruby_version = '>= 3.1' + + s.files = [ + '.gitignore', + 'MIT-LICENSE', + 'README.md', + 'Rakefile', + 'bin/mysql2postgres', + 'lib/mysql2postgres.rb', + 'lib/mysql2postgres/converter.rb', + 'lib/mysql2postgres/connection.rb', + 'lib/mysql2postgres/mysql_reader.rb', + 'lib/mysql2postgres/postgres_db_writer.rb', + 'lib/mysql2postgres/postgres_file_writer.rb', + 'lib/mysql2postgres/postgres_db_writer.rb', + 'lib/mysql2postgres/postgres_writer.rb', + 'lib/mysql2postgres/version.rb', + 'mysql2postgres.gemspec', + 'test/fixtures/config_all_options.yml', + 'test/fixtures/config_min_options.yml', + 'test/fixtures/config_to_file.yml', + 'test/fixtures/seed_integration_tests.sql', + 'test/integration/convert_to_db_test.rb', + 'test/integration/convert_to_file_test.rb', + 'test/integration/converter_test.rb', + 'test/integration/mysql_reader_connection_test.rb', + 'test/integration/mysql_reader_test.rb', + 'test/integration/postgres_db_writer_test.rb', + 'test/units/mysql_test.rb', + 'test/units/option_test.rb', + 'test/units/postgres_file_writer_test.rb', + 'test/test_helper.rb' + ] + s.homepage = 'https://github.com/AlphaNodes/mysql2postgres' + s.rdoc_options = ['--charset=UTF-8'] + s.require_paths = ['lib'] + s.summary = 'MySQL to PostgreSQL Data Translation' + + s.add_dependency 'pg', '~> 1.5.3' + s.add_dependency 'rake' + s.add_dependency 'ruby-mysql', '~> 3.0.1' +end diff --git a/mysqltopostgres.gemspec b/mysqltopostgres.gemspec deleted file mode 100644 index dedccbc..0000000 --- a/mysqltopostgres.gemspec +++ /dev/null @@ -1,95 +0,0 @@ - -Gem::Specification.new do |s| - s.name = 'mysqltopostgres' - s.version = '0.3.1' - s.licenses = ['MIT'] - - s.authors = [ - 'Max Lapshin ', - 'Anton Ageev ', - 'Samuel Tribehou ', - 'Marco Nenciarini ', - 'James Nobis ', - 'quel ', - 'Holger Amann ', - 'Maxim Dobriakov ', - 'Michael Kimsal ', - 'Jacob Coby ', - 'Neszt Tibor ', - 'Miroslav Kratochvil ', - 'Paul Gallagher ', - 'Alex C Jokela ', - 'Peter Clark ', - 'Juga Paazmaya ' - ] - s.date = '2015-11-26' - s.default_executable = 'mysqltopostgres' - s.description = 'Translates MySQL -> PostgreSQL' - s.email = 'paazmaya@yahoo.com' - s.executables = ['mysqltopostgres'] - - s.files = [ - '.gitignore', - 'MIT-LICENSE', - 'README.md', - 'Rakefile', - 'bin/mysqltopostgres', - 'lib/mysqltopostgres.rb', - 'lib/mysql2psql/config.rb', - 'lib/mysql2psql/config_base.rb', - 'lib/mysql2psql/converter.rb', - 'lib/mysql2psql/connection.rb', - 'lib/mysql2psql/errors.rb', - 'lib/mysql2psql/mysql_reader.rb', - 'lib/mysql2psql/postgres_db_writer.rb', - 'lib/mysql2psql/postgres_file_writer.rb', - 'lib/mysql2psql/postgres_db_writer.rb', - 'lib/mysql2psql/postgres_writer.rb', - 'lib/mysql2psql/version.rb', - 'lib/mysql2psql/writer.rb', - 'mysqltopostgres.gemspec', - 'test/fixtures/config_all_options.yml', - 'test/fixtures/seed_integration_tests.sql', - 'test/integration/convert_to_db_test.rb', - 'test/integration/convert_to_file_test.rb', - 'test/integration/converter_test.rb', - 'test/integration/mysql_reader_base_test.rb', - 'test/integration/mysql_reader_test.rb', - 'test/integration/postgres_db_writer_base_test.rb', - 'test/lib/ext_test_unit.rb', - 'test/lib/test_helper.rb', - 'test/units/config_base_test.rb', - 'test/units/config_test.rb', - 'test/units/postgres_file_writer_test.rb' - ] - s.homepage = 'https://github.com/maxlapshin/mysql2postgres' - s.rdoc_options = ['--charset=UTF-8'] - s.require_paths = ['lib'] - s.rubygems_version = '2.4.0' - s.summary = 'MySQL to PostgreSQL Data Translation' - s.test_files = [ - 'test/integration/convert_to_db_test.rb', - 'test/integration/convert_to_file_test.rb', - 'test/integration/converter_test.rb', - 'test/integration/mysql_reader_base_test.rb', - 'test/integration/mysql_reader_test.rb', - 'test/integration/postgres_db_writer_base_test.rb', - 'test/lib/ext_test_unit.rb', - 'test/lib/test_helper.rb', - 'test/units/config_base_test.rb', - 'test/units/config_test.rb', - 'test/units/postgres_file_writer_test.rb' - ] - - s.add_dependency('mysql-pr', ['~> 2.9']) - s.add_dependency('postgres-pr', ['~> 0.6']) - s.add_dependency('test-unit', ['~> 2.1']) - - if RUBY_PLATFORM == 'java' - s.add_dependency('activerecord', ['~> 3.2']) - s.add_dependency('jdbc-postgres', ['~> 9.4']) - s.add_dependency('activerecord-jdbc-adapter', ['~> 1.2']) - s.add_dependency('activerecord-jdbcpostgresql-adapter', ['~> 1.2']) - end - -end diff --git a/test/fixtures/config_all_options.yml b/test/fixtures/config_all_options.yml index 43629cb..fe29960 100644 --- a/test/fixtures/config_all_options.yml +++ b/test/fixtures/config_all_options.yml @@ -1,20 +1,22 @@ mysql: - hostname: localhost - port: 3306 - socket: /tmp/mysql.sock - username: somename - password: secretpassword - database: somename - -destination: - # if file is given, output goes to file, else postgres - file: somefile - postgres: - hostname: localhost - port: 5432 - username: somename - password: secretpassword + hostname: 127.0.0.1 + port: 3306 + username: root + password: BestPasswordEver database: somename + encoding: utf8mb4 + +# if MYSQL2POSTGRES_ENV and RAILS_ENV is missing, development is used! +destinations: + development: + adapter: file + filename: /tmp/exported_file.sql + test: + hostname: localhost + port: 5432 + username: postgres + password: postgres + database: somename # if tables is given, only the listed tables will be converted. leave empty to convert all tables. tables: @@ -28,8 +30,10 @@ exclude_tables: - table5 - table6 +dump_file_directory: /tmp + # if suppress_data is true, only the schema definition will be exported/migrated, and not the data -suppress_data: true +suppress_data: false # if suppress_ddl is true, only the data will be exported/imported, and not the schema suppress_ddl: false diff --git a/test/fixtures/config_min_options.yml b/test/fixtures/config_min_options.yml new file mode 100644 index 0000000..08c8893 --- /dev/null +++ b/test/fixtures/config_min_options.yml @@ -0,0 +1,16 @@ +mysql: + hostname: localhost + username: root + password: BestPasswordEver + database: somename + +# if MYSQL2POSTGRES_ENV and RAILS_ENV is missing, development is used! +destinations: + test: + hostname: localhost + port: 5432 + username: somename + password: secretpassword + database: somename + +# all options are false as default diff --git a/test/fixtures/config_to_file.yml b/test/fixtures/config_to_file.yml new file mode 100644 index 0000000..7f2fc36 --- /dev/null +++ b/test/fixtures/config_to_file.yml @@ -0,0 +1,31 @@ +mysql: + hostname: 127.0.0.1 + port: 3306 + username: root + password: BestPasswordEver + database: somename + encoding: utf8mb4 + +# if MYSQL2POSTGRES_ENV and RAILS_ENV is missing, development is used! +destinations: + test: + adapter: file + filename: /tmp/exported_file.sql + +# if tables is given, only the listed tables will be converted. leave empty to convert all tables. +tables: +- table1 +- table2 +- table3 +- table4 + +dump_file_directory: /tmp + +# if suppress_data is true, only the schema definition will be exported/migrated, and not the data +suppress_data: false + +# if suppress_ddl is true, only the data will be exported/imported, and not the schema +suppress_ddl: false + +# if force_truncate is true, forces a table truncate before table loading +force_truncate: false diff --git a/test/integration/convert_to_db_test.rb b/test/integration/convert_to_db_test.rb index 34c65fa..fee43f9 100644 --- a/test/integration/convert_to_db_test.rb +++ b/test/integration/convert_to_db_test.rb @@ -1,28 +1,30 @@ -require 'test_helper' +# frozen_string_literal: true -require 'mysqltopostgres' +require File.expand_path '../test_helper', __dir__ class ConvertToDbTest < Test::Unit::TestCase class << self def startup seed_test_database - @@options = get_test_config_by_label(:localmysql_to_db_convert_all) - @@mysql2psql = Mysql2psql.new(@@options) - @@mysql2psql.convert - @@mysql2psql.writer.open - end - - def shutdown - @@mysql2psql.writer.close end end + def setup + @mysql2postgres = instance_from_file + @options = @mysql2postgres.options + @options[:force_truncate] = true + @options.delete :tables # convert all available tables + + @mysql2postgres.convert + @mysql2postgres.writer.connection.open end def teardown + @mysql2postgres&.writer&.connection&.finish end def test_table_creation - assert_true @@mysql2psql.writer.exists?('numeric_types_basics') + tables = @mysql2postgres.writer.connection.tables + assert tables.include?('numeric_types_basics') end end diff --git a/test/integration/convert_to_file_test.rb b/test/integration/convert_to_file_test.rb index 4301923..35cdf29 100644 --- a/test/integration/convert_to_file_test.rb +++ b/test/integration/convert_to_file_test.rb @@ -1,74 +1,83 @@ -require 'test_helper' +# frozen_string_literal: true -require 'mysqltopostgres' +require File.expand_path '../test_helper', __dir__ class ConvertToFileTest < Test::Unit::TestCase + attr_reader :content + class << self def startup - seed_test_database - @@options = get_test_config_by_label(:localmysql_to_file_convert_all) - @@mysql2psql = Mysql2psql.new(@@options) - @@mysql2psql.convert - @@content = IO.read(@@mysql2psql.options.destfile) + seed_test_database option_file: 'config_to_file' end - - end - def setup end - def teardown - end + def setup + @mysql2postgres = instance_from_file 'config_to_file' + @options = @mysql2postgres.options + @options[:force_truncate] = true + @options.delete :tables # convert all available tables - def content - @@content + @mysql2postgres.convert + @content = File.read @mysql2postgres.options[:destination][:filename] end def test_table_creation assert_not_nil content.match('DROP TABLE IF EXISTS "numeric_types_basics" CASCADE') - assert_not_nil content.match(/CREATE TABLE "numeric_types_basics"/) + assert_not_nil content.include?('CREATE TABLE "numeric_types_basics"') end def test_basic_numerics_tinyint - assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_tinyint" smallint,.*\)', Regexp::MULTILINE).match(content) + assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_tinyint" smallint,.*\)', Regexp::MULTILINE) + .match(content) end def test_basic_numerics_smallint - assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_smallint" integer,.*\)', Regexp::MULTILINE).match(content) + assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_smallint" integer,.*\)', Regexp::MULTILINE) + .match(content) end def test_basic_numerics_mediumint - assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_mediumint" integer,.*\)', Regexp::MULTILINE).match(content) + assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_mediumint" integer,.*\)', Regexp::MULTILINE) + .match(content) end def test_basic_numerics_int - assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_int" integer,.*\)', Regexp::MULTILINE).match(content) + assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_int" integer,.*\)', Regexp::MULTILINE) + .match(content) end def test_basic_numerics_integer - assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_integer" integer,.*\)', Regexp::MULTILINE).match(content) + assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_integer" integer,.*\)', Regexp::MULTILINE) + .match(content) end def test_basic_numerics_bigint - assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_bigint" bigint,.*\)', Regexp::MULTILINE).match(content) + assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_bigint" bigint,.*\)', Regexp::MULTILINE) + .match(content) end def test_basic_numerics_real - assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_real" double precision,.*\)', Regexp::MULTILINE).match(content) + assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_real" double precision,.*\)', Regexp::MULTILINE) + .match(content) end def test_basic_numerics_double - assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_double" double precision,.*\)', Regexp::MULTILINE).match(content) + assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_double" double precision,.*\)', Regexp::MULTILINE) + .match(content) end def test_basic_numerics_float - assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_float" double precision,.*\)', Regexp::MULTILINE).match(content) + assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_float" double precision,.*\)', Regexp::MULTILINE) + .match(content) end def test_basic_numerics_decimal - assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_decimal" numeric\(10, 0\),.*\)', Regexp::MULTILINE).match(content) + assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_decimal" numeric\(10, 0\),.*\)', Regexp::MULTILINE) + .match(content) end def test_basic_numerics_numeric - assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_numeric" numeric\(10, 0\)[\w\n]*\)', Regexp::MULTILINE).match(content) + assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_numeric" numeric\(10, 0\)[\w\n]*\)', Regexp::MULTILINE) + .match(content) end end diff --git a/test/integration/converter_test.rb b/test/integration/converter_test.rb index f44b79a..f4860e3 100644 --- a/test/integration/converter_test.rb +++ b/test/integration/converter_test.rb @@ -1,30 +1,27 @@ -require 'test_helper' +# frozen_string_literal: true -require 'mysql2psql/converter' +require File.expand_path '../test_helper', __dir__ class ConverterTest < Test::Unit::TestCase class << self def startup - seed_test_database - @@options = get_test_config_by_label(:localmysql_to_file_convert_nothing) + seed_test_database option_file: 'config_to_file' end - - end - def setup end - def teardown - end + def setup + @options = options_from_file 'config_to_file' + @options[:suppress_data] = true + @options[:suppress_ddl] = true - def options - @@options + @destfile = get_temp_file 'mysql2postgres_test' end def test_new_converter assert_nothing_raised do - reader = get_test_reader(options) - writer = get_test_file_writer(options) - converter = Mysql2psql::Converter.new(reader, writer, options) + reader = get_test_reader @options + writer = Mysql2postgres::PostgresFileWriter.new @destfile, @options[:destination] + converter = Mysql2postgres::Converter.new reader, writer, @options assert_equal 0, converter.convert end end diff --git a/test/integration/mysql_reader_base_test.rb b/test/integration/mysql_reader_base_test.rb deleted file mode 100644 index 58c054c..0000000 --- a/test/integration/mysql_reader_base_test.rb +++ /dev/null @@ -1,43 +0,0 @@ -require 'test_helper' - -require 'mysql2psql/mysql_reader' - -class MysqlReaderBaseTest < Test::Unit::TestCase - class << self - def startup - seed_test_database - @@options = get_test_config_by_label(:localmysql_to_file_convert_nothing) - end - - end - def setup - end - - def teardown - end - - def options - @@options - end - - def test_mysql_connection - assert_nothing_raised do - reader = Mysql2psql::MysqlReader.new(options) - end - end - - def test_mysql_reconnect - assert_nothing_raised do - reader = Mysql2psql::MysqlReader.new(options) - reader.reconnect - end - end - - def test_mysql_connection_without_port - assert_nothing_raised do - options.mysqlport = '' - options.mysqlsocket = '' - reader = Mysql2psql::MysqlReader.new(options) - end - end -end diff --git a/test/integration/mysql_reader_connection_test.rb b/test/integration/mysql_reader_connection_test.rb new file mode 100644 index 0000000..0bbab3f --- /dev/null +++ b/test/integration/mysql_reader_connection_test.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +require File.expand_path '../test_helper', __dir__ + +class MysqlReaderConnectionTest < Test::Unit::TestCase + class << self + def startup + seed_test_database option_file: 'config_to_file' + end + end + + def setup + @options = options_from_file 'config_to_file' + end + + def test_mysql_connection + assert_nothing_raised do + Mysql2postgres::MysqlReader.new @options + end + end + + def test_mysql_reconnect + assert_nothing_raised do + reader = Mysql2postgres::MysqlReader.new @options + reader.reconnect + end + end + + def test_mysql_connection_without_port + assert_nothing_raised do + @options[:mysql][:port] = '' + @options[:mysql][:socket] = '' + Mysql2postgres::MysqlReader.new @options + end + end +end diff --git a/test/integration/mysql_reader_test.rb b/test/integration/mysql_reader_test.rb index 0b7adc5..193b870 100644 --- a/test/integration/mysql_reader_test.rb +++ b/test/integration/mysql_reader_test.rb @@ -1,32 +1,27 @@ -require 'test_helper' +# frozen_string_literal: true + +require File.expand_path '../test_helper', __dir__ class MysqlReaderTest < Test::Unit::TestCase class << self def startup - seed_test_database - @@options = get_test_config_by_label(:localmysql_to_file_convert_nothing) - @@reader = get_test_reader(@@options) + seed_test_database option_file: 'config_to_file' end - - end - def setup end - def teardown - end - - def reader - @@reader + def setup + @options = options_from_file 'config_to_file' + @reader = get_test_reader @options end def test_db_connection assert_nothing_raised do - reader.mysql.ping + @reader.mysql.ping end end def test_tables_collection - values = reader.tables.select { |t| t.name == 'numeric_types_basics' } + values = @reader.tables.select { |t| t.name == 'numeric_types_basics' } assert_true values.length == 1 assert_equal 'numeric_types_basics', values[0].name end @@ -34,11 +29,11 @@ def test_tables_collection def test_paginated_read expected_rows = 3 page_size = 2 - expected_pages = (1.0 * expected_rows / page_size).ceil + # expected_pages = (1.0 * expected_rows / page_size).ceil row_count = my_row_count = 0 - table = reader.tables.select { |t| t.name == 'numeric_types_basics' }[0] - reader.paginated_read(table, page_size) do |_row, counter| + table = @reader.tables.find { |t| t.name == 'numeric_types_basics' } + @reader.paginated_read table, page_size do |_row, counter| row_count = counter my_row_count += 1 end diff --git a/test/integration/postgres_db_writer_base_test.rb b/test/integration/postgres_db_writer_base_test.rb deleted file mode 100644 index 86e9235..0000000 --- a/test/integration/postgres_db_writer_base_test.rb +++ /dev/null @@ -1,28 +0,0 @@ -require 'test_helper' - -require 'mysql2psql/postgres_db_writer' - -class PostgresDbWriterBaseTest < Test::Unit::TestCase - class << self - def startup - seed_test_database - @@options = get_test_config_by_label(:localmysql_to_db_convert_nothing) - end - - end - def setup - end - - def teardown - end - - def options - @@options - end - - def test_pg_connection - assert_nothing_raised do - reader = Mysql2psql::PostgresDbWriter.new(options) - end - end -end diff --git a/test/integration/postgres_db_writer_test.rb b/test/integration/postgres_db_writer_test.rb new file mode 100644 index 0000000..7b0e834 --- /dev/null +++ b/test/integration/postgres_db_writer_test.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +require File.expand_path '../test_helper', __dir__ + +class PostgresDbWriterTest < Test::Unit::TestCase + class << self + def startup + seed_test_database + end + end + + def setup + @options = options_from_file + @options[:suppress_data] = true + @options[:suppress_ddl] = true + end + + def test_pg_connection + assert_nothing_raised do + Mysql2postgres::PostgresDbWriter.new Tempfile.new('mysql2postgres_test_').path, + @options[:destination] + end + end +end diff --git a/test/lib/ext_test_unit.rb b/test/lib/ext_test_unit.rb deleted file mode 100644 index 072f300..0000000 --- a/test/lib/ext_test_unit.rb +++ /dev/null @@ -1,26 +0,0 @@ -module Test::Unit - class TestCase - def self.must(name, &block) - test_name = "test_#{name.gsub(/\s+/, '_')}".to_sym - defined = instance_method(test_name) rescue false - fail "#{test_name} is already defined in #{self}" if defined - if block_given? - define_method(test_name, &block) - else - define_method(test_name) do - flunk "No implementation provided for #{name}" - end - end - end - end -end - -module Test::Unit::Assertions - def assert_false(object, message = '') - assert_equal(false, object, message) - end - - def assert_true(object, message = '') - assert_equal(true, object, message) - end -end diff --git a/test/lib/test_helper.rb b/test/lib/test_helper.rb deleted file mode 100644 index 9437852..0000000 --- a/test/lib/test_helper.rb +++ /dev/null @@ -1,81 +0,0 @@ -require 'rubygems' -begin - gem 'test-unit' - require 'test/unit' -rescue LoadError - # assume using stdlib Test:Unit - require 'test/unit' -end - -require 'ext_test_unit' - -def seed_test_database - options = get_test_config_by_label(:localmysql_to_file_convert_nothing) - seedfilepath = "#{File.dirname(__FILE__)}/../fixtures/seed_integration_tests.sql" - rc = system("mysql -u#{options.mysqlusername} #{options.mysqldatabase} < #{seedfilepath}") - fail StandardError unless rc - return true -rescue - raise StandardError.new('Failed to seed integration test db. See README for setup requirements.') -end - -def get_test_reader(options) - require 'mysql2psql/mysql_reader' - Mysql2psql::MysqlReader.new(options) -rescue - raise StandardError.new('Failed to initialize integration test db. See README for setup requirements.') -end - -def get_test_file_writer(options) - require 'mysql2psql/postgres_file_writer' - Mysql2psql::PostgresFileWriter.new(options.destfile) -rescue => e - puts e.inspect - raise StandardError.new("Failed to initialize file writer from #{options.inspect}. See README for setup requirements.") -end - -def get_test_converter(options) - require 'mysql2psql/converter' - reader = get_test_reader(options) - writer = get_test_file_writer(options) - Mysql2psql::Converter.new(reader, writer, options) -rescue - raise StandardError.new("Failed to initialize converter from #{options.inspect}. See README for setup requirements.") -end - -def get_temp_file(basename) - require 'tempfile' - f = Tempfile.new(basename) - path = f.path - f.close! - path -end - -def get_new_test_config(to_file = true, include_tables = [], exclude_tables = [], suppress_data = false, suppress_ddl = false, force_truncate = false) - require 'mysql2psql/config' - require 'mysql2psql/config_base' - to_filename = to_file ? get_temp_file('mysql2psql_tmp_output') : nil - configtext = Mysql2psql::Config.template(to_filename, include_tables, exclude_tables, suppress_data, suppress_ddl, force_truncate) - configfile = get_temp_file('mysql2psql_tmp_config') - File.open(configfile, 'w:UTF-8') { |f| f.write(configtext) } - yaml = YAML.load_file configfile - Mysql2psql::ConfigBase.new(yaml) -rescue - raise StandardError.new("Failed to initialize options from #{configfile}. See README for setup requirements.") -end - -def get_test_config_by_label(name) - case name - when :localmysql_to_file_convert_nothing - get_new_test_config(true, ['unobtainium'], ['kryptonite'], true, true, false) - when :localmysql_to_file_convert_all - get_new_test_config(true, [], [], false, false, false) - when :localmysql_to_db_convert_all - get_new_test_config(false, [], [], false, false, false) - when :localmysql_to_db_convert_nothing - get_new_test_config(false, ['unobtainium'], ['kryptonite'], true, true, false) - else - fail StandardError.new("Invalid label: #{name}") - end -end - diff --git a/test/test_helper.rb b/test/test_helper.rb new file mode 100644 index 0000000..4cd77d0 --- /dev/null +++ b/test/test_helper.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +require 'rubygems' +require 'test/unit' +require 'debug' if ENV.fetch('https://github.com/ENABLE_DEBUG', nil) == '1' + +require File.expand_path('lib/mysql2postgres') + +def load_yaml_file(file = 'config_all_options') + YAML.load_file "#{__dir__}/fixtures/#{file}.yml" +end + +def instance_from_file(file = 'config_all_options') + Mysql2postgres.new load_yaml_file(file) +end + +def options_from_file(file = 'config_all_options') + instance_from_file(file).options +end + +def seed_test_database(option_file: 'config_all_options', sql_file: 'seed_integration_tests.sql') + options = options_from_file option_file + seedfilepath = File.expand_path "test/fixtures/#{sql_file}" + system 'mysql ' \ + "--host #{options[:mysql][:hostname]} " \ + "--port #{options[:mysql][:port]} " \ + "-u#{options[:mysql][:username]} " \ + "-p#{options[:mysql][:password]} " \ + "#{options[:mysql][:database]} < #{seedfilepath}", exception: true +rescue StandardError + raise 'Failed to seed integration test db. See README for setup requirements.' +end + +def get_test_reader(options) + Mysql2postgres::MysqlReader.new options +rescue StandardError + raise 'Failed to initialize integration test db. See README for setup requirements.' +end + +def get_temp_file(basename) + require 'tempfile' + f = Tempfile.new basename + path = f.path + f.close! + path +end diff --git a/test/units/config_base_test.rb b/test/units/config_base_test.rb deleted file mode 100644 index 9498f2e..0000000 --- a/test/units/config_base_test.rb +++ /dev/null @@ -1,48 +0,0 @@ -require 'test_helper' -require 'mysqltopostgres' - -# -# -class ConfigBaseTest < Test::Unit::TestCase - attr_reader :config, :yaml - def setup - @yaml = YAML.load_file "#{File.dirname(__FILE__)}/../fixtures/config_all_options.yml" - @config = Mysql2psql::ConfigBase.new(yaml) - end - - def teardown - @config = nil - end - - def test_config_loaded - assert_not_nil config.config - assert_equal yaml, config.config - end - - def test_uninitialized_error_when_not_found_and_no_default - assert_raises(Mysql2psql::UninitializedValueError) do - value = @config.not_found(:none) - end - end - - def test_default_when_not_found - expected = 'defaultvalue' - value = @config.not_found(expected) - assert_equal expected, value - end - - def test_mysql_hostname - value = @config.mysqlhostname - assert_equal 'localhost', value - end - - def test_mysql_hostname_array_access - value = @config[:mysqlhostname] - assert_equal 'localhost', value - end - - def test_dest_file - value = @config.destfile - assert_equal 'somefile', value - end -end diff --git a/test/units/config_test.rb b/test/units/config_test.rb deleted file mode 100644 index 85fc706..0000000 --- a/test/units/config_test.rb +++ /dev/null @@ -1,16 +0,0 @@ -require 'test_helper' - -require 'mysql2psql/config' - -class ConfigTest < Test::Unit::TestCase - attr_reader :config_all_opts - def setup - @config_all_opts = YAML.load_file "#{File.dirname(__FILE__)}/../fixtures/config_all_options.yml" - end - - def test_config_loaded - value = Mysql2psql::Config.new(config_all_opts) - assert_not_nil value - end - -end diff --git a/test/units/mysql_test.rb b/test/units/mysql_test.rb new file mode 100644 index 0000000..764c929 --- /dev/null +++ b/test/units/mysql_test.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +require File.expand_path '../test_helper', __dir__ + +class MysqlTest < Test::Unit::TestCase + def test_mysql_charset + charset = ::Mysql::Charset.by_number 192 + assert_equal 'utf8mb3', charset.name + end +end diff --git a/test/units/option_test.rb b/test/units/option_test.rb new file mode 100644 index 0000000..aa1136d --- /dev/null +++ b/test/units/option_test.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +require File.expand_path '../test_helper', __dir__ +require 'yaml' + +class SettingTest < Test::Unit::TestCase + def test_options_loaded + options = options_from_file + + assert_equal false, options[:suppress_data] + assert_equal 'postgres', options[:destination][:username] + assert_equal 'somename', options[:destination][:database] + end +end diff --git a/test/units/postgres_file_writer_test.rb b/test/units/postgres_file_writer_test.rb index a2d4a7a..a455408 100644 --- a/test/units/postgres_file_writer_test.rb +++ b/test/units/postgres_file_writer_test.rb @@ -1,23 +1,26 @@ -require 'test_helper' +# frozen_string_literal: true -require 'mysqltopostgres' +require File.expand_path '../test_helper', __dir__ class PostgresFileWriterTest < Test::Unit::TestCase attr_accessor :destfile + def setup - @destfile = get_temp_file('mysql2psql_test_destfile') - rescue => e - raise StandardError.new('Failed to initialize integration test db. See README for setup requirements.') + @destfile = get_temp_file 'mysql2postgres_test_destfile' + rescue StandardError + raise 'Failed to initialize integration test db. See README for setup requirements.' end def teardown - File.delete(destfile) if File.exist?(destfile) + File.delete destfile end - def test_basic_write - writer = Mysql2psql::PostgresFileWriter.new(destfile) + def test_file_writer + destination = { filename: '/tmp/test.sql' } + writer = Mysql2postgres::PostgresFileWriter.new @destfile, destination writer.close - content = IO.read(destfile) + content = File.read destfile + assert_not_nil content.match("SET client_encoding = 'UTF8'") assert_nil content.match('unobtanium') end