diff --git a/.Rbuildignore b/.Rbuildignore index 6c1e194..67b498d 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -11,3 +11,4 @@ _pkgdown.yml ^inst/tests$ TODO.md ^.lintr$ +^CRAN-SUBMISSION$ diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index 6dc366a..3e91f34 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -33,11 +33,11 @@ jobs: steps: - uses: actions/checkout@v2 - - uses: r-lib/actions/setup-r@v1 + - uses: r-lib/actions/setup-r@v2 with: r-version: ${{ matrix.config.r }} - - uses: r-lib/actions/setup-pandoc@v1 + - uses: r-lib/actions/setup-pandoc@v2 - name: Query dependencies run: | diff --git a/DESCRIPTION b/DESCRIPTION index 105320e..b9d7567 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,12 +1,12 @@ Package: bcputility Type: Package Title: Wrapper for SQL Server bcp Utility -Version: 0.4.0 +Version: 0.4.2 Authors@R: person("Thomas", "Roh", email = "thomas@roh.engineering", role = c("aut", "cre")) Description: Provides functions to utilize a command line utility that does bulk inserts and exports from SQL Server databases. License: MIT + file LICENSE Encoding: UTF-8 -RoxygenNote: 7.2.2 +RoxygenNote: 7.2.3 SystemRequirements: bcp Utility Depends: R (>= 3.5.0) Imports: data.table, sf, methods diff --git a/NEWS.md b/NEWS.md index 4d69d84..a1b0f1c 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,10 @@ +# bcputility 0.4.2 + +* Added ability to turn on QUOTED_IDENTIFIERS. This is required for some +Microsoft database products. Use the bcp "-q" option. + +* Column names are quoted so that reserved words can be used. + # bcputility 0.4.0 * Override/set path to sqlcmd with diff --git a/R/bcp.R b/R/bcp.R index 54d02ad..31a27c3 100644 --- a/R/bcp.R +++ b/R/bcp.R @@ -105,6 +105,7 @@ bcpImport <- function( bcp <- findUtility('bcp') # syntax differs for the two utilities bcpArgs <- mapConnectArgs(connectargs = connectargs, utility = 'bcp') + quotedIdentifiers <- connectargs[['quotedidentifiers']] isSpatial <- methods::is(x, 'sf') if (methods::is(x, 'data.frame')) { tmp <- tempfile(fileext = '.dat') @@ -141,10 +142,10 @@ bcpImport <- function( bcpArgs <- append(bcpArgs, list('-t', shQuote(fieldterminator), '-r', shQuote(rowterminator), '-c')) - bcpArgs <- append(bcpArgs, list(quoteTable(table = table), + bcpArgs <- append(bcpArgs, list( + table = table, 'in', shQuote(fileName)), after = 0) - tableExists <- checkTableExists(connectargs = connectargs, - table = table) + tableExists <- checkTableExists(connectargs = connectargs, table = table) append <- tableExists && isFALSE(overwrite) if (isFALSE(append)) { # guess sql server data types @@ -163,8 +164,8 @@ bcpImport <- function( } } # create empty table - createOutput <- createTable(connectargs = connectargs, - table = table, coltypes = dbTypes, stderr = TRUE) + createOutput <- createTable(connectargs = connectargs, table = table, + coltypes = dbTypes, stderr = TRUE) if (length(createOutput) != 0) { stop(paste(createOutput, collapse = ' ')) } @@ -432,10 +433,11 @@ createTable <- function(connectargs, table, coltypes, ...) { query <- sprintf( 'CREATE TABLE %s (%s);', quotedTable, - paste(names(coltypes), coltypes, sep = ' ', collapse = ', ') + paste(sprintf('[%s]', names(coltypes)), coltypes, sep = ' ', + collapse = ', ') ) - sqlcmdArgs <- append(mapConnectArgs(connectargs = connectargs, - utility = 'sqlcmd'), values = list('-Q', shQuote(query))) + sqlcmdArgs <- mapConnectArgs(connectargs = connectargs, utility = 'sqlcmd') + sqlcmdArgs <- append(sqlcmdArgs, values = list('-Q', shQuote(query))) system2(command = sqlcmd, args = sqlcmdArgs, ...) } #' @rdname createTable @@ -446,8 +448,8 @@ dropTable <- function(connectargs, table, ...) { sqlcmd <- findUtility('sqlcmd') quotedTable <- quoteTable(table) query <- sprintf('DROP TABLE %s;', quotedTable) - sqlcmdArgs <- append(mapConnectArgs(connectargs = connectargs, - utility = 'sqlcmd'), values = list('-Q', shQuote(query))) + sqlcmdArgs <- mapConnectArgs(connectargs = connectargs, utility = 'sqlcmd') + sqlcmdArgs <- append(sqlcmdArgs, values = list('-Q', shQuote(query))) system2(command = sqlcmd, args = sqlcmdArgs, ...) } #' @rdname createTable @@ -456,21 +458,14 @@ dropTable <- function(connectargs, table, ...) { #' checkTableExists <- function(connectargs, table) { sqlcmd <- findUtility('sqlcmd') - # IF OBJECT_ID('*objectName*', 'U') IS NOT NULL - quotedTable <- quoteTable(table) - quotedTable <- strsplit(x = quotedTable, split = '\\.')[[1]] - if (length(quotedTable) > 2) { - stop('Only `` and `.
` are supported for table - argument.') - } - if (length(quotedTable) < 2) { - quotedTable <- append(quotedTable, '[dbo]', after = 0) - } - query <- sprintf('EXECUTE sp_tables @table_name = %s, @table_owner = %s', - quotedTable[2], quotedTable[1]) - sqlcmdArgs <- append(mapConnectArgs(connectargs = connectargs, - utility = 'sqlcmd'), values = list('-Q', shQuote(query))) - system2(command = sqlcmd, args = sqlcmdArgs, stdout = TRUE)[[3]] != '' + query <- sprintf(" + IF OBJECT_ID('%s', 'U') IS NOT NULL + BEGIN PRINT 1 END + ELSE + BEGIN PRINT 0 END", unQuoteTable(table)) + sqlcmdArgs <- mapConnectArgs(connectargs = connectargs, utility = 'sqlcmd') + sqlcmdArgs <- append(sqlcmdArgs, values = list('-Q', shQuote(query))) + identical(system2(command = sqlcmd, args = sqlcmdArgs, stdout = TRUE)[[1]], '1') } readTable <- function(connectargs, table, ...) { sqlcmd <- findUtility('sqlcmd') @@ -478,8 +473,8 @@ readTable <- function(connectargs, table, ...) { query <- sprintf('SET NOCOUNT ON; SELECT * FROM %s;', quotedTable) queryHeaders <- sprintf('SET NOCOUNT ON; SELECT TOP 0 * FROM %s;', quotedTable) - sqlcmdArgs <- append(mapConnectArgs(connectargs = connectargs, - utility = 'sqlcmd'), + sqlcmdArgs <- mapConnectArgs(connectargs = connectargs, utility = 'sqlcmd') + sqlcmdArgs <- append(sqlcmdArgs, values = list( '-s', shQuote(','), '-W', @@ -526,13 +521,19 @@ readTable <- function(connectargs, table, ...) { #' use Azure Active Directory authentication, does not work with integrated #' authentication. #' +#' @param quotedidentifiers +#' +#' set QUOTED_IDENTIFIERS option to 'ON' for the connection between bcp/sqlcmd +#' and SQL Server. +#' #' @return #' #' a list with connection arguments #' #' @export makeConnectArgs <- function(server, database, username, password, - trustedconnection = TRUE, trustservercert = FALSE, azure = FALSE) { + trustedconnection = TRUE, trustservercert = FALSE, azure = FALSE, + quotedidentifiers = FALSE) { if (isTRUE(trustedconnection) && isTRUE(azure)) { stop('trustedconnection and azure cannot both be TRUE') } @@ -551,6 +552,10 @@ makeConnectArgs <- function(server, database, username, password, if (isTRUE(trustservercert)) { connectArgs <- append(x = connectArgs, values = list(trustservercert = trustservercert)) + } + if (isTRUE(quotedidentifiers)) { + connectArgs <- append(x = connectArgs, + values = list(quotedidentifiers = quotedidentifiers)) } connectArgs } @@ -567,6 +572,17 @@ quoteTable <- function(table) { sprintf('[%s]', x) }), collapse = '.') } +unQuoteTable <- function(table) { + paste(lapply(strsplit(table, split = '\\.')[[1]], function(x) { + if (substring(text = x, first = 1, last = 1) == '[' && + substring(text = x, + first = nchar(x), + last = nchar(x)) == ']') { + return(substring(text = x, first = 2, last = nchar(x) - 1)) + } + x + }), collapse = '.') +} convertGeoCol <- function(connectargs, table, geometrycol, binarycol, spatialtype, srid, ...) { sqlcmd <- findUtility(utility = 'sqlcmd') @@ -600,14 +616,16 @@ mapConnectArgs <- function(connectargs, utility = c('sqlcmd', 'bcp') username = '-U', password = '-P', azure = '-G', - trustservercert = '-C'), + trustservercert = '-C', + quotedidentifiers = '-I'), bcp = list(server = '-S', database = '-d', trustedconnection = '-T', username = '-U', password = '-P', azure = '-G', - trustservercert = '-u'), + trustservercert = '-u', + quotedidentifiers = '-q'), stop('Unsupported utility') ) argSyntax <- argSyntax[names(connectargs)] diff --git a/R/sysdata.rda b/R/sysdata.rda index 789ec29..99acb77 100644 Binary files a/R/sysdata.rda and b/R/sysdata.rda differ diff --git a/docs/404.html b/docs/404.html index ed6b215..eec35d9 100644 --- a/docs/404.html +++ b/docs/404.html @@ -39,7 +39,7 @@ bcputility - 0.4.0 + 0.4.2 diff --git a/docs/LICENSE-text.html b/docs/LICENSE-text.html index 353b9cf..58d6c6f 100644 --- a/docs/LICENSE-text.html +++ b/docs/LICENSE-text.html @@ -17,7 +17,7 @@ bcputility - 0.4.0 + 0.4.2 diff --git a/docs/LICENSE.html b/docs/LICENSE.html index 33e5ee9..f440ff9 100644 --- a/docs/LICENSE.html +++ b/docs/LICENSE.html @@ -17,7 +17,7 @@ bcputility - 0.4.0 + 0.4.2 diff --git a/docs/authors.html b/docs/authors.html index ef8fbbc..f196ad6 100644 --- a/docs/authors.html +++ b/docs/authors.html @@ -17,7 +17,7 @@ bcputility - 0.4.0 + 0.4.2 @@ -61,14 +61,14 @@

Citation

-

Roh T (2023). +

Roh T (2024). bcputility: Wrapper for SQL Server bcp Utility. https://bcputility.roh.engineering, https://github.com/tomroh/bcputility.

@Manual{,
   title = {bcputility: Wrapper for SQL Server bcp Utility},
   author = {Thomas Roh},
-  year = {2023},
+  year = {2024},
   note = {https://bcputility.roh.engineering, https://github.com/tomroh/bcputility},
 }
diff --git a/docs/index.html b/docs/index.html index c3ab272..9c80e14 100644 --- a/docs/index.html +++ b/docs/index.html @@ -40,7 +40,7 @@ bcputility - 0.4.0 + 0.4.2 @@ -125,11 +125,11 @@

Import as.POSIXct('2022-12-31 23:59:59'), by = 'min'), size = n, replace = TRUE) ) connectArgs <- makeConnectArgs(server = server, database = database) -con <- DBI::dbConnect(odbc::odbc(), +con <- DBI::dbConnect(odbc::odbc(), Driver = "SQL Server", Server = server, Database = database) -importResults <- microbenchmark::microbenchmark( +importResults <- microbenchmark::microbenchmark( bcpImport1000 = { bcpImport(importTable, connectargs = connectArgs, @@ -163,7 +163,7 @@

Import stdout = FALSE) }, dbWriteTable = { - con <- DBI::dbConnect(odbc::odbc(), + con <- DBI::dbConnect(odbc::odbc(), Driver = driver, Server = server, Database = database, @@ -255,7 +255,7 @@

Export Tabledata.table”. Optimized write formats for date times from fwrite outperforms bcp for data that is small enough to be pulled into memory.

-exportResults <- microbenchmark::microbenchmark(
+exportResults <- microbenchmark::microbenchmark(
   bcpExportChar = {
     bcpExport('inst/benchmarks/test1.csv',
               connectargs = connectArgs,
@@ -271,7 +271,7 @@ 

Export Table= FALSE) }, fwriteQuery = { - fwrite(DBI::dbReadTable(con, 'importTableInit'), + fwrite(DBI::dbReadTable(con, 'importTableInit'), 'inst/benchmarks/test3.csv', dateTimeAs = 'write.csv', col.names = FALSE) }, @@ -340,7 +340,7 @@

Export Query
 query <- 'SELECT * FROM [dbo].[importTable1] WHERE int < 1000'
-queryResults <- microbenchmark::microbenchmark(
+queryResults <- microbenchmark::microbenchmark(
   bcpExportQueryChar = {
     bcpExport('inst/benchmarks/test4.csv',
               connectargs = connectArgs,
@@ -356,7 +356,7 @@ 

Export Query= FALSE) }, fwriteQuery = { - fwrite(DBI::dbGetQuery(con, query), + fwrite(DBI::dbGetQuery(con, query), 'inst/benchmarks/test6.csv', dateTimeAs = 'write.csv', col.names = FALSE) }, @@ -431,7 +431,7 @@

Import Geometryshp1 <- cbind(nc[sample.int(nrow(nc), n / divN, replace = TRUE),], importTable[seq_len(n / divN), ], id = seq_len(n / divN)) -geometryResults <- microbenchmark::microbenchmark( +geometryResults <- microbenchmark::microbenchmark( bcpImportGeometry = { bcpImport(shp1, connectargs = connectArgs, @@ -442,7 +442,7 @@

Import Geometry bcpOptions = list("-b", 50000, "-a", 4096, "-m", 0)) }, odbcImportGeometry = { - con <- DBI::dbConnect(odbc::odbc(), + con <- DBI::dbConnect(odbc::odbc(), driver = driver, server = server, database = database, @@ -452,11 +452,11 @@

Import Geometry geometryColumn <- 'geom' binaryColumn <- 'geomWkb' srid <- sf::st_crs(nc)$epsg - shpBin2 <- data.table(shp1) - data.table::set(x = shpBin2, j = binaryColumn, + shpBin2 <- data.table(shp1) + data.table::set(x = shpBin2, j = binaryColumn, value = blob::new_blob(lapply(sf::st_as_binary(shpBin2[[geometryColumn]]), as.raw))) - data.table::set(x = shpBin2, j = geometryColumn, value = NULL) + data.table::set(x = shpBin2, j = geometryColumn, value = NULL) dataTypes <- DBI::dbDataType(con, shpBin2) dataTypes[binaryColumn] <- 'varbinary(max)' DBI::dbWriteTable(conn = con, name = tableName, value = shpBin2, @@ -480,7 +480,7 @@

Import Geometry bcpOptions = list("-b", 50000, "-a", 4096, "-m", 0)) }, odbcImportGeography = { - con <- DBI::dbConnect(odbc::odbc(), + con <- DBI::dbConnect(odbc::odbc(), driver = driver, server = server, database = database, @@ -490,11 +490,11 @@

Import Geometry geometryColumn <- 'geom' binaryColumn <- 'geomWkb' srid <- sf::st_crs(nc)$epsg - shpBin4 <- data.table(shp1) - data.table::set(x = shpBin4, j = binaryColumn, + shpBin4 <- data.table(shp1) + data.table::set(x = shpBin4, j = binaryColumn, value = blob::new_blob(lapply(sf::st_as_binary(shpBin4[[geometryColumn]]), as.raw))) - data.table::set(x = shpBin4, j = geometryColumn, value = NULL) + data.table::set(x = shpBin4, j = geometryColumn, value = NULL) dataTypes <- DBI::dbDataType(con, shpBin4) dataTypes[binaryColumn] <- 'varbinary(max)' DBI::dbWriteTable(conn = con, name = tableName, value = shpBin4, diff --git a/docs/news/index.html b/docs/news/index.html index e46fd7f..5cc4bfe 100644 --- a/docs/news/index.html +++ b/docs/news/index.html @@ -17,7 +17,7 @@ bcputility - 0.4.0 + 0.4.2

@@ -48,7 +48,12 @@

Changelog

- + +
  • Added ability to turn on QUOTED_IDENTIFIERS. This is required for some Microsoft database products. Use the bcp “-q” option.

  • +
  • Column names are quoted so that reserved words can be used.

  • +
+
+
  • Override/set path to sqlcmd with options(bcputility.sqlcmd.path = "<path-to-sqlcmd>")

  • Added bcpVersion and sqlcmdVersion to check versions.

  • Specify tables without schema strictly as a character vector of size 1 e.g. "<schema>.<table>". The schema and table will be quoted although special characters are not recommended except for “_“.

  • @@ -74,7 +79,7 @@
    • set the path to the bcp utility with the bcputility.bcp.path option

    • added links to source code, bug reports, and documentation site

    • -
    • rowterminator and fieldterminator are no passed to data.table::fwrite if data is in memory

    • +
    • rowterminator and fieldterminator are no passed to data.table::fwrite if data is in memory

    • Added support for geometry/geography data import for ‘sf’ objects

diff --git a/docs/pkgdown.yml b/docs/pkgdown.yml index 8eaf8b1..33a1275 100644 --- a/docs/pkgdown.yml +++ b/docs/pkgdown.yml @@ -1,6 +1,6 @@ -pandoc: 2.19.2 +pandoc: 3.1.1 pkgdown: 2.0.7 pkgdown_sha: ~ articles: {} -last_built: 2023-01-16T22:28Z +last_built: 2024-04-02T13:57Z diff --git a/docs/reference/SQLServerCLIVersions.html b/docs/reference/SQLServerCLIVersions.html index 69af34c..e020a3e 100644 --- a/docs/reference/SQLServerCLIVersions.html +++ b/docs/reference/SQLServerCLIVersions.html @@ -17,7 +17,7 @@ bcputility - 0.4.0 + 0.4.2
diff --git a/docs/reference/bcpExport.html b/docs/reference/bcpExport.html index 8a2b793..b608288 100644 --- a/docs/reference/bcpExport.html +++ b/docs/reference/bcpExport.html @@ -18,7 +18,7 @@ bcputility - 0.4.0 + 0.4.2 diff --git a/docs/reference/bcpImport.html b/docs/reference/bcpImport.html index a81ddc0..791dc96 100644 --- a/docs/reference/bcpImport.html +++ b/docs/reference/bcpImport.html @@ -18,7 +18,7 @@ bcputility - 0.4.0 + 0.4.2 @@ -118,7 +118,7 @@

Value

Details

-

If x is a dataframe object, data.table::fwrite is used to write the +

If x is a dataframe object, data.table::fwrite is used to write the in memory object to disk in a temporary file that is deleted when the function exits. The fieldterminator and rowterminator are ignored in this case.

diff --git a/docs/reference/createTable.html b/docs/reference/createTable.html index 6831243..753982c 100644 --- a/docs/reference/createTable.html +++ b/docs/reference/createTable.html @@ -17,7 +17,7 @@ bcputility - 0.4.0 + 0.4.2
diff --git a/docs/reference/index.html b/docs/reference/index.html index c38dc4c..33ef9f0 100644 --- a/docs/reference/index.html +++ b/docs/reference/index.html @@ -17,7 +17,7 @@ bcputility - 0.4.0 + 0.4.2 @@ -59,23 +59,23 @@

All functions

- + - + - +

Import data to SQL Server

-

createTable() dropTable() checkTableExists()

+

bcpVersion() sqlcmdVersion()

Create or drop table

Check bcp and sqlcmd versions

-

mapDataTypes() varChar() varBinary() int()

+

createTable() dropTable() checkTableExists()

Determine SQL Server data types from data frame. Follows SQL Server -data type size constraints and chooses the smallest data type size.

Create or drop table

makeConnectArgs()

Create a named list of connection arguments to translate to bcp and sqlcmd options

-

bcpVersion() sqlcmdVersion()

+

mapDataTypes() varChar() varBinary() int()

Check bcp and sqlcmd versions

Determine SQL Server data types from data frame. Follows SQL Server +data type size constraints and chooses the smallest data type size.

@@ -65,7 +65,8 @@

Create a named list of connection arguments to translate to bcp and password, trustedconnection = TRUE, trustservercert = FALSE, - azure = FALSE + azure = FALSE, + quotedidentifiers = FALSE ) @@ -99,6 +100,11 @@

Arguments

use Azure Active Directory authentication, does not work with integrated authentication.

+ +
quotedidentifiers
+

set QUOTED_IDENTIFIERS option to 'ON' for the connection between bcp/sqlcmd +and SQL Server.

+

Value

diff --git a/docs/reference/mapDataTypes.html b/docs/reference/mapDataTypes.html index cddaeb4..5e86402 100644 --- a/docs/reference/mapDataTypes.html +++ b/docs/reference/mapDataTypes.html @@ -20,7 +20,7 @@ bcputility - 0.4.0 + 0.4.2
diff --git a/docs/sitemap.xml b/docs/sitemap.xml index f7bf88d..6d3a725 100644 --- a/docs/sitemap.xml +++ b/docs/sitemap.xml @@ -10,16 +10,16 @@ /reference/bcpExport.html - /reference/createTable.html + /reference/SQLServerCLIVersions.html - /reference/mapDataTypes.html + /reference/createTable.html /reference/makeConnectArgs.html - /reference/SQLServerCLIVersions.html + /reference/mapDataTypes.html /news/index.html diff --git a/man/makeConnectArgs.Rd b/man/makeConnectArgs.Rd index 841ecdf..8f8647e 100644 --- a/man/makeConnectArgs.Rd +++ b/man/makeConnectArgs.Rd @@ -12,7 +12,8 @@ makeConnectArgs( password, trustedconnection = TRUE, trustservercert = FALSE, - azure = FALSE + azure = FALSE, + quotedidentifiers = FALSE ) } \arguments{ @@ -30,6 +31,9 @@ makeConnectArgs( \item{azure}{use Azure Active Directory authentication, does not work with integrated authentication.} + +\item{quotedidentifiers}{set QUOTED_IDENTIFIERS option to 'ON' for the connection between bcp/sqlcmd +and SQL Server.} } \value{ a list with connection arguments diff --git a/tests/a-init-db.R b/tests/a-init-db.R index ea2a6c3..00c3d4b 100644 --- a/tests/a-init-db.R +++ b/tests/a-init-db.R @@ -6,7 +6,7 @@ library(bcputility) # variables MSSQL_SERVER and MSSQL_DB or modify the `makeConnectArgs` function # below. It is best to `source` this file from the package home directory. -# Writes the data to SQL Server if TRUES +# Writes the data to SQL Server if TRUE initDB <- FALSE # This should never need to be TRUE as the internall R data is already contained # in the package. diff --git a/tests/b-import-db.R b/tests/b-import-db.R index 71d55be..441fc4a 100644 --- a/tests/b-import-db.R +++ b/tests/b-import-db.R @@ -237,4 +237,5 @@ if (identical(Sys.getenv('NOT_CRAN'), 'true') && isTRUE(testLocal)) { database <- Sys.getenv('MSSQL_DB') connectArgs <- makeConnectArgs(server = server, database = database) testImport(connectargs = connectArgs) + print('All import tests passed') } diff --git a/tests/c-export-db.R b/tests/c-export-db.R index cbd9392..3033703 100644 --- a/tests/c-export-db.R +++ b/tests/c-export-db.R @@ -72,4 +72,5 @@ if (identical(Sys.getenv('NOT_CRAN'), 'true') && isTRUE(testLocal)) { database <- Sys.getenv('MSSQL_DB') connectArgs <- makeConnectArgs(server = server, database = database) testExport(connectargs = connectArgs) + print('All export tests passed') } diff --git a/tests/d-types.R b/tests/d-types.R index c8888f4..916af66 100644 --- a/tests/d-types.R +++ b/tests/d-types.R @@ -40,3 +40,4 @@ testTypeMapping <- function() { stopifnot() } testTypeMapping() +print('All data type tests passed')