From f1bac1a308c4f60dc649ec2eb4772140bf46bf07 Mon Sep 17 00:00:00 2001 From: Mikhail Titov Date: Fri, 11 Oct 2019 18:26:09 -0500 Subject: [PATCH 1/9] Visual Studio project --- .gitignore | 4 +++ odbc_fdw.vcxproj | 77 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) create mode 100644 odbc_fdw.vcxproj diff --git a/.gitignore b/.gitignore index f134aaa..2db6ff9 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,7 @@ test/config/*.tar # JIT **/*.bc +x64/ +Debug/ +Release/ +.vs/ diff --git a/odbc_fdw.vcxproj b/odbc_fdw.vcxproj new file mode 100644 index 0000000..1457375 --- /dev/null +++ b/odbc_fdw.vcxproj @@ -0,0 +1,77 @@ + + + + + Debug + Win32 + + + Release + Win32 + + + Debug + x64 + + + Release + x64 + + + + {2EB5E1F5-F370-4367-9BAD-CD241B28434C} + Win32Proj + odbc_fdw + v140 + DynamicLibrary + Link + true + $(ProgramFiles) + $(ProgramW6432) + + + 11 + $(pf)\PostgreSQL\$(pgversion) + + + + + + $(pgroot)\include;$(pgroot)\include\server;$(pgroot)\include\server\port\win32;$(pgroot)\include\server\port\win32_msvc;$(IncludePath) + $(pgroot)\lib;$(LibraryPath) + + + true + + + + Level3 + CompileAsC + false + AdvancedVectorExtensions2 + ProgramDatabase + + + Speed + true + AnySuitable + true + + + Disabled + true + true + MultiThreadedDebugDLL + true + DEBUG;%(PreprocessorDefinitions) + + + postgres.lib;%(AdditionalDependencies) + true + true + + + + + + \ No newline at end of file From ad2ed4d9bf98b989a49b7170d710a7e6b147770e Mon Sep 17 00:00:00 2001 From: Mikhail Titov Date: Fri, 11 Oct 2019 18:22:32 -0500 Subject: [PATCH 2/9] MSVC: Use _stricmp in place of strcasecmp --- odbc_fdw.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/odbc_fdw.c b/odbc_fdw.c index 8d974d8..2578a70 100644 --- a/odbc_fdw.c +++ b/odbc_fdw.c @@ -61,6 +61,10 @@ #include "access/table.h" #endif +#if defined(_WIN32) +#define strcasecmp _stricmp +#endif + /* TupleDescAttr was backported into 9.5.9 and 9.6.5 but we support any 9.5.X */ #ifndef TupleDescAttr #define TupleDescAttr(tupdesc, i) ((tupdesc)->attrs[(i)]) From 307b10d66ce503886252b61f61486be3783618bb Mon Sep 17 00:00:00 2001 From: Mikhail Titov Date: Mon, 14 Oct 2019 16:16:02 -0500 Subject: [PATCH 3/9] Explicitly export symbols on Windows using PGDLLEXPORT --- odbc_fdw.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/odbc_fdw.c b/odbc_fdw.c index 2578a70..c52a96a 100644 --- a/odbc_fdw.c +++ b/odbc_fdw.c @@ -177,11 +177,11 @@ typedef enum { TEXT_CONVERSION, HEX_CONVERSION, BIN_CONVERSION, BOOL_CONVERSION /* * SQL functions */ -extern Datum odbc_fdw_handler(PG_FUNCTION_ARGS); -extern Datum odbc_fdw_validator(PG_FUNCTION_ARGS); -extern Datum odbc_tables_list(PG_FUNCTION_ARGS); -extern Datum odbc_table_size(PG_FUNCTION_ARGS); -extern Datum odbc_query_size(PG_FUNCTION_ARGS); +PGDLLEXPORT Datum odbc_fdw_handler(PG_FUNCTION_ARGS); +PGDLLEXPORT Datum odbc_fdw_validator(PG_FUNCTION_ARGS); +PGDLLEXPORT Datum odbc_tables_list(PG_FUNCTION_ARGS); +PGDLLEXPORT Datum odbc_table_size(PG_FUNCTION_ARGS); +PGDLLEXPORT Datum odbc_query_size(PG_FUNCTION_ARGS); PG_FUNCTION_INFO_V1(odbc_fdw_handler); PG_FUNCTION_INFO_V1(odbc_fdw_validator); From 91118b896a7699aed7ceefa0b78b4b4de0c2a96a Mon Sep 17 00:00:00 2001 From: Mikhail Titov Date: Mon, 14 Oct 2019 16:40:01 -0500 Subject: [PATCH 4/9] Appveyor --- appveyor.yml | 102 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 appveyor.yml diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 0000000..41bdeec --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,102 @@ +pull_requests: + do_not_increment_build_number: true +os: Visual Studio 2017 +configuration: Release +platform: x64 +clone_depth: 1 +services: + - mysql +environment: + PGUSER: postgres + PGPASSWORD: Password12! + matrix: + - pg: 11.5-2 + PlatformToolset: v140 + - pg: 12.0-1 + PlatformToolset: v141 + +init: # Make %exe% available for caching +- ps: | + if ("$env:PLATFORM" -eq "x64") { + $env:pf = "$env:ProgramFiles" + $env:x64 = "-x64" + } else { + $env:pf = "${env:ProgramFiles(x86)}" + } + $env:exe = "postgresql-$env:pg-windows$env:x64.exe" + [Environment]::SetEnvironmentVariable("exe", $env:exe, "Machine") + +install: +- ps: | + # http://www.databasesoup.com/2016/05/changing-postgresql-version-numbering.html + $env:pgversion = $env:pg -replace "\.\d+-\d+$", "" + # [Environment]::SetEnvironmentVariable("pgversion", $env:pgversion, "Machine") + $env:pgroot = "$env:pf\PostgreSQL\$env:pgversion" + if ("$env:pg" -notmatch "\.") { + $env:Path += ";C:\msys64\usr\bin;C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\amd64" + git clone -q --depth 1 --branch $env:pg https://git.postgresql.org/git/postgresql.git c:\projects\postgresql + pushd c:\projects\postgresql + perl src\tools\msvc\build.pl + perl src\tools\msvc\install.pl "$env:pgroot" + popd + } else { + if (-not (Test-Path "$env:pgroot\bin")) { + if (-not (Test-Path "$env:exe")) { + Start-FileDownload "http://get.enterprisedb.com/postgresql/$env:exe" + } + & ".\$env:exe" --unattendedmodeui none --mode unattended --superpassword "$env:PGPASSWORD" --servicepassword "$env:PGPASSWORD" | Out-Null + Stop-Service "postgresql$env:x64-$env:pgversion" + } + } + +cache: +- '%exe%' + +build_script: +- msbuild /p:PlatformToolset=%PlatformToolset% /p:configuration=%CONFIGURATION% /p:platform=%PLATFORM% + odbc_fdw.vcxproj + /verbosity:minimal /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" + +after_build: +- ps: | + $dll = ($env:PLATFORM -replace "x86", ".") + "\$env:CONFIGURATION\odbc_fdw.dll" + Add-AppveyorMessage Packing -Category Information + New-Item -Name tmp\share\extension -ItemType "directory" + Copy-Item "*.sql" -Destination tmp\share\extension\ + Copy-Item "*.control" -Destination tmp\share\extension\ + Copy-Item LICENSE -Destination tmp\ODBC_FDW_LICENSE + New-Item -Name tmp\lib -ItemType "directory" + New-Item -Name tmp\debug_symbols -ItemType "directory" + Copy-Item $dll -Destination tmp\lib + Copy-Item ($dll -replace "\.dll", ".pdb") -Destination tmp\debug_symbols + if ($env:APPVEYOR_REPO_TAG -eq "false") { + $env:APPVEYOR_REPO_TAG_NAME=${env:APPVEYOR_REPO_COMMIT}.SubString(0,8) + } + $zip = "odbc_fdw-$env:APPVEYOR_REPO_TAG_NAME-pg$env:pgversion-windows-$env:PLATFORM.zip" + Compress-Archive -Path ".\tmp\*" -DestinationPath $zip + +test_script: +- ps: | + $env:path += ";$env:pgroot\bin" + Add-AppveyorMessage "Copying the extension files to the PostgreSQL directories." -Category Information + Expand-Archive -LiteralPath $zip -DestinationPath $env:pgroot + if ("$env:pg" -notmatch "\.") { + Set-Content -path pg.pass -value "$env:pgpassword" -encoding ascii + initdb -A md5 -U "$env:PGUSER" --pwfile=pg.pass C:\pgdata + pg_ctl register -S demand -N "postgresql$env:x64-$env:pgversion" -D c:\pgdata + } + Start-Service postgresql$env:x64-$env:pgversion +# & .\test\appveyor_tests.ps1 + +artifacts: +- path: '*.zip' + +deploy: + # Deploy to GitHub Releases + - provider: GitHub + draft: false + prerelease: false + auth_token: + secure: QR7uEDoDDcUclQhaQDdbymUotMh1Gq++D+ljzKGC4a1Y9hr6BszPYRcLy9i0HJ/k + on: + appveyor_repo_tag: true # deploy on tag push only From 7043276e349ee6238888c07e3b6948eec87b43e7 Mon Sep 17 00:00:00 2001 From: Mikhail Titov Date: Mon, 14 Oct 2019 17:24:47 -0500 Subject: [PATCH 5/9] AppVeyor badge for README --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 64d8125..3a7e477 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ -ODBC FDW for PostgreSQL 9.5+ [![Build Status](https://travis-ci.org/CartoDB/odbc_fdw.svg?branch=master)](https://travis-ci.org/CartoDB/odbc_fdw) +ODBC FDW for PostgreSQL 9.5+ ============================ +[![Travis Build Status](https://travis-ci.org/CartoDB/odbc_fdw.svg?branch=master)](https://travis-ci.org/CartoDB/odbc_fdw) +[![AppVeyor Build Status](https://ci.appveyor.com/api/projects/status/github/mlt/odbc_fdw?branch=appveyor&svg=true)](https://ci.appveyor.com/project/PurpleI2P/odbc-fdw "Get your fresh Windows build here!") This PostgreSQL extension implements a Foreign Data Wrapper (FDW) for remote databases using Open Database Connectivity [ODBC](http://msdn.microsoft.com/en-us/library/ms714562(v=VS.85).aspx). From efb2c4abbd019e2415d5649ce3d90b9e617ab3e6 Mon Sep 17 00:00:00 2001 From: Mikhail Titov Date: Mon, 14 Oct 2019 18:08:38 -0500 Subject: [PATCH 6/9] AppVeyor CI tests --- appveyor.yml | 14 +++++++- test/appveyor_tests.ps1 | 78 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 test/appveyor_tests.ps1 diff --git a/appveyor.yml b/appveyor.yml index 41bdeec..3648580 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -6,14 +6,23 @@ platform: x64 clone_depth: 1 services: - mysql + - mssql2017 environment: PGUSER: postgres PGPASSWORD: Password12! + MYSQL_PWD: Password12! + SQLCMDPASSWORD: Password12! matrix: - pg: 11.5-2 PlatformToolset: v140 - pg: 12.0-1 PlatformToolset: v141 + - pg: master + PlatformToolset: v141 + configuration: Debug +matrix: + allow_failures: + - pg: master init: # Make %exe% available for caching - ps: | @@ -86,7 +95,7 @@ test_script: pg_ctl register -S demand -N "postgresql$env:x64-$env:pgversion" -D c:\pgdata } Start-Service postgresql$env:x64-$env:pgversion -# & .\test\appveyor_tests.ps1 + & .\test\appveyor_tests.ps1 artifacts: - path: '*.zip' @@ -100,3 +109,6 @@ deploy: secure: QR7uEDoDDcUclQhaQDdbymUotMh1Gq++D+ljzKGC4a1Y9hr6BszPYRcLy9i0HJ/k on: appveyor_repo_tag: true # deploy on tag push only + +# on_finish: +# - ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) \ No newline at end of file diff --git a/test/appveyor_tests.ps1 b/test/appveyor_tests.ps1 new file mode 100644 index 0000000..6fd1b5c --- /dev/null +++ b/test/appveyor_tests.ps1 @@ -0,0 +1,78 @@ +# Taken from https://deadroot.info/scripts/2018/09/04/PowerShell-Templating +function Merge-Tokens($template, $tokens) +{ + return [regex]::Replace( + $template, + '\$\{(?\w+)\}', + { + param($match) + $tokenName = $match.Groups['tokenName'].Value + return $tokens[$tokenName] + }) +} + +# https://www.appveyor.com/docs/services-databases +$Config = @{ + mysql = @{ + driver = 'MySQL ODBC 5.3 Unicode Driver' + host = 'localhost' + port = '3306' + dbname = 'fdw_tests' + encoding = 'utf8' + user = 'root' + password = 'Password12!' + } + postgres = @{ + driver = 'PostgreSQL Unicode(x64)' + host = 'localhost' + port = '5432' + dbname = 'fdw_tests' + user = 'postgres' + password = 'Password12!' + } + sqlserver = @{ + driver = 'ODBC Driver 13 for SQL Server' + host = '(local)\SQL2017' + port = '1433' + dbname = 'master' + user = 'sa' + password = 'Password12!' + } +} + +foreach ($c in $Config.GetEnumerator()) { + $tpl = Get-Content "$PSScriptRoot\template\$($c.Name)_installation_test.tpl" -Raw + $generated_test = Merge-Tokens $tpl $($c.Value) + Set-Content -Path "$PSScriptRoot\sql/$($c.Name)_10_installation_test.sql" -Value $generated_test + Set-Content -Path "$PSScriptRoot\expected\$($c.Name)_10_installation_test.out" -Value $generated_test +} + +$env:Path += ";C:\Program Files\MySQL\MySQL Server 5.7\bin" +& mysql -e "create database fdw_tests character set utf8mb4 collate utf8mb4_unicode_ci;" --user=root +& cmd.exe /c 'mysql fdw_tests --user=root < test\fixtures\mysql_fixtures.sql' +& createdb fdw_tests +& psql -f test\fixtures\postgres_fixtures.sql fdw_tests postgres 2>&1 | + %{ if ($_ -is [System.Management.Automation.ErrorRecord]) { $_.Exception.Message } else { $_ } } | + Out-Default +& sqlcmd -S "(local)\SQL2017" -U "sa" -d master -i "$PSScriptRoot\fixtures\sqlserver_fixtures.sql" +Rename-Item -Path "$PSScriptRoot\sql\sqlserver_20_query_test_disabled.sql" -NewName "sqlserver_20_query_test.sql" + +Add-AppveyorTest Regression -Framework pg_regress -FileName sql\ -Outcome Running +$env:Outcome="Passed" +$elapsed=(Measure-Command { + pg_regress "--bindir=$env:pgroot\bin" --inputdir=test --outputdir=test --load-extension=odbc_fdw --dbname=regression ` + mysql_10_installation_test mysql_20_query_test ` + postgres_10_installation_test postgres_20_query_test ` + sqlserver_10_installation_test sqlserver_20_query_test ` + 2>&1 | + %{ if ($_ -is [System.Management.Automation.ErrorRecord]) { $_.Exception.Message } else { $_ } } | + Out-Default + if ($LASTEXITCODE -ne 0) { + $env:Outcome="Failed" + } +}).TotalMilliseconds +Update-AppVeyorTest Regression -Framework pg_regress -FileName sql\ -Outcome "$env:Outcome" -Duration $elapsed +if ("$env:Outcome" -ne "Passed") { + type test\regression.diffs + $host.SetShouldExit($LastExitCode) +} From f1a2a338fc69d9d702b5088d5b32a9d2674331df Mon Sep 17 00:00:00 2001 From: Mikhail Titov Date: Mon, 21 Oct 2019 16:36:31 -0500 Subject: [PATCH 7/9] Use newer v12 PostgreSQL ODBC driver --- appveyor.yml | 1 + test/appveyor_tests.ps1 | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index 3648580..0b62e95 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -60,6 +60,7 @@ install: cache: - '%exe%' +- psqlodbc_x64.msi build_script: - msbuild /p:PlatformToolset=%PlatformToolset% /p:configuration=%CONFIGURATION% /p:platform=%PLATFORM% diff --git a/test/appveyor_tests.ps1 b/test/appveyor_tests.ps1 index 6fd1b5c..9f8d203 100644 --- a/test/appveyor_tests.ps1 +++ b/test/appveyor_tests.ps1 @@ -57,6 +57,13 @@ $env:Path += ";C:\Program Files\MySQL\MySQL Server 5.7\bin" & sqlcmd -S "(local)\SQL2017" -U "sa" -d master -i "$PSScriptRoot\fixtures\sqlserver_fixtures.sql" Rename-Item -Path "$PSScriptRoot\sql\sqlserver_20_query_test_disabled.sql" -NewName "sqlserver_20_query_test.sql" +if (-not (Test-Path "$PSScriptRoot\..\psqlodbc_x64.msi")) { + Start-FileDownload "https://ftp.postgresql.org/pub/odbc/versions/msi/psqlodbc_12_00_0000-x64.zip" + Expand-Archive -LiteralPath psqlodbc_12_00_0000-x64.zip -DestinationPath . + Remove-Item psqlodbc_12_00_0000-x64.zip +} +& msiexec /i psqlodbc_x64.msi /qn /quiet + Add-AppveyorTest Regression -Framework pg_regress -FileName sql\ -Outcome Running $env:Outcome="Passed" $elapsed=(Measure-Command { From eb3734fab02d0f88e18bcfe64401419755e35696 Mon Sep 17 00:00:00 2001 From: Mikhail Titov Date: Tue, 15 Oct 2019 18:03:52 -0500 Subject: [PATCH 8/9] Fix failing SQL Server test --- odbc_fdw.c | 1 + 1 file changed, 1 insertion(+) diff --git a/odbc_fdw.c b/odbc_fdw.c index c52a96a..d9f3cdd 100644 --- a/odbc_fdw.c +++ b/odbc_fdw.c @@ -2040,6 +2040,7 @@ odbcImportForeignSchema(ImportForeignSchemaStmt *stmt, Oid serverOid) * in that case we must avoid using an schema to query the table columns. */ schema_name = NULL; + missing_foreign_schema = false; } /* Since we haven't specified SQL_ALL_CATALOGS in the From b445cb2280e89fea5ed4fc379f7e634345b6ff3c Mon Sep 17 00:00:00 2001 From: Mikhail Titov Date: Tue, 15 Oct 2019 18:54:10 -0500 Subject: [PATCH 9/9] Fix lnext: Lists have been converted to arrays https://git.postgresql.org/gitweb/?p=postgresql.git;a=commitdiff;h=1cff1b95ab6ddae32faa3efe0d95a820dbfdc164 --- odbc_fdw.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/odbc_fdw.c b/odbc_fdw.c index d9f3cdd..6c1a81e 100644 --- a/odbc_fdw.c +++ b/odbc_fdw.c @@ -2155,7 +2155,11 @@ odbcImportForeignSchema(ImportForeignSchemaStmt *stmt, Oid serverOid) int option_count = 0; const char *prefix = empty_string_if_null(options.prefix); +#if PG_VERSION_NUM >= 130000 + table_columns_cell = lnext(table_columns, table_columns_cell); +#else table_columns_cell = lnext(table_columns_cell); +#endif initStringInfo(&create_statement); appendStringInfo(&create_statement, "CREATE FOREIGN TABLE \"%s\".\"%s%s\" (", stmt->local_schema, prefix, (char *) table_name);