Skip to content

Commit

Permalink
Fix problems due to differing prefixes in S3 stores (#11)
Browse files Browse the repository at this point in the history
Use temporary symlink to always make a valid store for reading in targets no matter the the object path.
  • Loading branch information
Noam Ross authored Nov 28, 2023
1 parent 3c98c2c commit 5bb5526
Showing 1 changed file with 32 additions and 20 deletions.
52 changes: 32 additions & 20 deletions R/tar_read_version.R
Original file line number Diff line number Diff line change
Expand Up @@ -54,49 +54,61 @@ tar_read_raw_version <- function(name, ref = "HEAD", repo = ".", store = NULL) {
}

target <- switch(record$repository,
local = read_target_aws(record, path_store = path_store),
aws = read_target_aws(record, path_store = path_store),
abort("Unknown targets repository type: ", record$repository)
local = read_target_aws(record),
aws = read_target_aws(record),
abort("Unknown targets repository type: ", record$repository)
)

target
}

read_target_aws <- function(record, path_store) {
read_target_aws <- function(record) {
aws_loc <- aws_loc_from_meta_path(record$path[[1]])
local_target_path <- get_file_version(
path = aws_loc$key, ref = aws_loc$version,
repo = paste0("s3://", aws_loc$bucket),
endpoint = aws_loc$endpoint, region = aws_loc$region
)
record_local <- record
record_local$path <- list(local_target_path)
record_local$repository <- "local"
targets::tar_read_raw(record_local$name,
meta = record_local,
store = path_dir(path_dir(local_target_path))
)
if (record$format == "file") {
return(local_target_path)
} else {
record_local <- record
record_local$path <- NA
record_local$repository <- "local"
temp_store <- path_dir(dir_create(path(file_temp("_targets"), "objects")))
link_create(local_target_path, path(temp_store, "objects", path_file(local_target_path)))
on.exit(dir_delete(temp_store))
return(targets::tar_read_raw(record_local$name,
meta = record_local,
store = temp_store))
}
}

read_target_local <- function(record, path_store) {
read_target_local <- function(record) {
# For local targets
local_target_path <- get_file_version(
path = record$path[[1]], ref = record$version,
repo = record$repository
)
record_local <- record
record_local$path <- list(local_target_path)

targets::tar_read_raw(record_local$name,
meta = record_local,
store = path_store
)
if (record$format == "file") {
return(local_target_path)
} else {
record_local <- record
record_local$path <- NA
record_local$repository <- "local"
temp_store <- path_dir(dir_create(path(file_temp("_targets"), "objects")))
link_create(local_target_path, path(temp_store, "objects", path_file(local_target_path)))
on.exit(dir_delete(temp_store))
return(targets::tar_read_raw(record_local$name,
meta = record_local,
store = temp_store))
}
}

aws_loc_from_meta_path <- function(path) {
splits <- strsplit(path, "=")
aws_loc <- structure(lapply(splits, function(x) x[[2]]),
.Names = vapply(splits, function(x) x[[1]], character(1))
.Names = vapply(splits, function(x) x[[1]], character(1))
)
if (!is.null(aws_loc$endpoint)) {
aws_loc$endpoint <- rawToChar(openssl::base64_decode(aws_loc$endpoint))
Expand Down

0 comments on commit 5bb5526

Please sign in to comment.