Skip to content

Commit

Permalink
Merge pull request #100 from jburke-cadc/CADC-12859
Browse files Browse the repository at this point in the history 
CADC-12859 added getResponseFormat method to get the archive return type
  • Loading branch information
@pdowler
pdowler authored Feb 12, 2024
2 parents 35865da + 81a4e8c commit b8596ab
Show file tree
Hide file tree
Showing 7 changed files with 111 additions and 39 deletions.
2 changes: 1 addition & 1 deletion cadc-pkg-server/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ sourceCompatibility = 1.8

group = 'org.opencadc'

version = '1.2.2'
version = '1.2.3'

description = 'OpenCADC CADC package server library'
def git_url = 'https://github.com/opencadc/dal'
Expand Down
6 changes: 3 additions & 3 deletions cadc-pkg-server/src/main/java/org/opencadc/pkg/server/PackageItem.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ public PackageItem(String relativePath) {
}

/**
* Creates a resource for a package. The relative path is used to create
* Creates a PackageItem for a file. The relative path is used to create
* the file structure inside a package.
*
* @param relativePath path to the resource in the package.
Expand All @@ -115,11 +115,11 @@ public PackageItem(String relativePath, URL content) {
}

/**
* Creates a resource for a package. The relative path is used to create
* Creates a PackageItem for a symbolic link. The relative path is used to create
* the file structure inside a package.
*
* @param relativePath path to the resource in the package.
* @param linkTarget path to the resource in the package.
* @param linkTarget relative path to the link target in the package.
*/
public PackageItem(String relativePath, String linkTarget) {
if (!StringUtil.hasText(relativePath)) {
Expand Down
30 changes: 22 additions & 8 deletions cadc-pkg-server/src/main/java/org/opencadc/pkg/server/PackageRunner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,10 +152,31 @@ public void run() {
log.info(logInfo.end());
}

/**
* Get the expected initial ExecutionPhase of the Job when the PackageRunner runs.
*
* @return ExecutionPhase
*/
protected ExecutionPhase getInitialPhase() {
return ExecutionPhase.QUEUED;
}

/**
* Get the MIME type of the package. Defaults to application/x-tar
* if the RESPONSEFORMAT job parameter is not provided.
*
* @return MIME type
*/
protected String getResponseFormat() {
// Package type is in RESPONSEFORMAT Job parameter (optional)
String responseFormat = ParameterUtil.findParameterValue("RESPONSEFORMAT", job.getParameterList());
if (!StringUtil.hasLength(responseFormat)) {
// Not provided, set default
responseFormat = TarWriter.MIME_TYPE;
}
return responseFormat;
}

private void doIt() {
ExecutionPhase ep;
PackageWriter writer = null;
Expand Down Expand Up @@ -254,19 +275,12 @@ private ByteCountOutputStream initOutputStream(String mimeType, String contentDi
private PackageWriter initWriter()
throws IOException {

// Package type is in RESPONSEFORMAT Job parameter (optional)
String responseFormat = ParameterUtil.findParameterValue("RESPONSEFORMAT", job.getParameterList());

if (!StringUtil.hasLength(responseFormat)) {
// Not provided, set default
responseFormat = TarWriter.MIME_TYPE;
}

StringBuilder cdisp = new StringBuilder();
cdisp.append("inline;filename=");
cdisp.append(packageName);

// Initialize the writer, underlying syncoutput and output streams.
String responseFormat = getResponseFormat();
if (responseFormat.equals(ZipWriter.MIME_TYPE)) {
cdisp.append(ZipWriter.EXTENSION);
this.bcOutputStream = initOutputStream(ZipWriter.MIME_TYPE, cdisp.toString());
Expand Down
11 changes: 9 additions & 2 deletions cadc-pkg-server/src/main/java/org/opencadc/pkg/server/PackageWriter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@
import java.util.Date;
import org.apache.commons.compress.archivers.ArchiveEntry;
import org.apache.commons.compress.archivers.ArchiveOutputStream;
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
import org.apache.log4j.Logger;

public abstract class PackageWriter {
Expand Down Expand Up @@ -141,8 +142,10 @@ public void write(PackageItem packageItem) throws IOException, InterruptedExcept
} else {
writeHTTPFile(packageItem);
}
} else {
} else if (packageItem.isSymbolicLink()) {
writeSymbolicLink(packageItem);
} else {
throw new IllegalArgumentException("Unknown PackageItem type: " + packageItem);
}
}

Expand Down Expand Up @@ -181,6 +184,7 @@ private void writeFile(PackageItem packageItem)
InputStream stream = fileURL.openStream();
MultiBufferIO multiBufferIO = new MultiBufferIO();
multiBufferIO.copy(stream, archiveOutputStream);
stream.close();
archiveOutputStream.closeArchiveEntry();
}

Expand Down Expand Up @@ -211,6 +215,7 @@ private void writeHTTPFile(PackageItem packageItem)
InputStream getIOStream = get.getInputStream();
MultiBufferIO multiBufferIO = new MultiBufferIO();
multiBufferIO.copy(getIOStream, archiveOutputStream);
getIOStream.close();
archiveOutputStream.closeArchiveEntry();
}

Expand All @@ -229,7 +234,9 @@ private void writeSymbolicLink(PackageItem packageItem)
archiveOutputStream.putArchiveEntry(archiveEntry);

// entry content is the symbolic link target path
archiveOutputStream.write(linkTarget.getBytes(StandardCharsets.UTF_8));
if (archiveEntry instanceof ZipArchiveEntry) {
archiveOutputStream.write(linkTarget.getBytes(StandardCharsets.UTF_8));
}
archiveOutputStream.closeArchiveEntry();
}

Expand Down
14 changes: 13 additions & 1 deletion cadc-pkg-server/src/main/java/org/opencadc/pkg/server/TarWriter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,10 @@

package org.opencadc.pkg.server;

import java.io.IOException;
import java.io.OutputStream;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Date;
import org.apache.commons.compress.archivers.ArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
Expand All @@ -88,6 +91,7 @@ public TarWriter(OutputStream ostream) {
((TarArchiveOutputStream)super.archiveOutputStream).setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX);
}

@Override
ArchiveEntry createFileEntry(String relativePath, long size, Date lastModifiedDate) {
log.debug(String.format("file ArchiveEntry: %s %s %s", relativePath, size, lastModifiedDate));

Expand All @@ -99,18 +103,26 @@ ArchiveEntry createFileEntry(String relativePath, long size, Date lastModifiedDa
return entry;
}

@Override
ArchiveEntry createDirectoryEntry(String relativePath) {
log.debug("directory ArchiveEntry: " + relativePath);

if (relativePath.startsWith("/")) {
relativePath = relativePath.substring(1);
}
if (!relativePath.endsWith("/")) {
relativePath = relativePath + "/";
}

return new TarArchiveEntry(relativePath, TarConstants.LF_DIR);
}

@Override
ArchiveEntry createSymbolicLinkEntry(String relativePath, String linkTarget) {
log.debug(String.format("symbolic link ArchiveEntry: %s -> %s", relativePath, linkTarget));

TarArchiveEntry entry = new TarArchiveEntry(relativePath, TarConstants.LF_SYMLINK);
entry.setLinkName(linkTarget);
entry.setSize(linkTarget.length());
return entry;
}

Expand Down
10 changes: 10 additions & 0 deletions cadc-pkg-server/src/main/java/org/opencadc/pkg/server/ZipWriter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@ public ZipWriter(OutputStream ostream) {
super(new ZipArchiveOutputStream(ostream));
}

@Override
ArchiveEntry createFileEntry(String relativePath, long size, Date lastModifiedDate) {
log.debug(String.format("file ArchiveEntry: %s %s %s", relativePath, size, lastModifiedDate));

Expand All @@ -100,14 +101,23 @@ ArchiveEntry createFileEntry(String relativePath, long size, Date lastModifiedDa
return entry;
}

@Override
ArchiveEntry createDirectoryEntry(String relativePath) {
log.debug("directory ArchiveEntry: " + relativePath);

if (relativePath.startsWith("/")) {
relativePath = relativePath.substring(1);
}
if (!relativePath.endsWith("/")) {
relativePath = relativePath + "/";
}

ZipArchiveEntry entry = new ZipArchiveEntry(relativePath);
entry.setUnixMode(UnixStat.DIR_FLAG);
return entry;
}

@Override
ArchiveEntry createSymbolicLinkEntry(String relativePath, String linkRelativePath) {
log.debug(String.format("symbolic link ArchiveEntry: %s -> %s", relativePath, linkRelativePath));

Expand Down
77 changes: 53 additions & 24 deletions cadc-pkg-server/src/test/java/org/opencadc/pkg/server/PackageWriterTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,25 +69,31 @@

package org.opencadc.pkg.server;

import ca.nrc.cadc.util.Log4jInit;
import java.io.File;
import java.io.FileOutputStream;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.junit.Assert;
import org.junit.Test;

public class PackageWriterTest {
private static final Logger log = Logger.getLogger(PackageWriterTest.class);

static {
Log4jInit.setLevel("org.opencadc.pkg.server", Level.INFO);
}

@Test
public void testCreateTarFile() {
try {
List<PackageItem> testPackageItems = getTestPackageItems();

File tarFile = File.createTempFile("tartest", ".tar");
log.debug("tar archive: " + tarFile.getAbsolutePath());
log.info("tar archive: " + tarFile.getAbsolutePath());
FileOutputStream fos = new FileOutputStream(tarFile);
TarWriter fw = new TarWriter(fos);
for (PackageItem pi : testPackageItems) {
Expand All @@ -112,7 +118,7 @@ public void testCreateZipFile() {
List<PackageItem> testPackageItems = getTestPackageItems();

File zipFile = File.createTempFile("ziptest", ".zip");
log.debug("zip archive: " + zipFile.getAbsolutePath());
log.info("zip archive: " + zipFile.getAbsolutePath());
FileOutputStream fos = new FileOutputStream(zipFile);
PackageWriter fw = new ZipWriter(fos);
for (PackageItem pi : testPackageItems) {
Expand All @@ -134,34 +140,57 @@ public void testCreateZipFile() {
protected List<PackageItem> getTestPackageItems() {
// Create PackageItems for testing
// Files are in test/resources
String dir1Path = "some/path/";
PackageItem dir1 = new PackageItem(dir1Path);
log.debug(dir1);
String some = "some/";
PackageItem someDir = new PackageItem(some);
log.debug(someDir);

String somePath = "some/path/";
PackageItem somePathDir = new PackageItem(somePath);
log.debug(somePathDir);

String someEmpty = "some/empty/";
PackageItem someEmptyDir = new PackageItem(someEmpty);
log.debug(someEmptyDir);

String someEmptyPath = "some/empty/path/";
PackageItem someEmptyPathDir = new PackageItem(someEmptyPath);
log.debug(someEmptyPathDir);

String govCanadaGif = "some/path/GovCanada.gif";
URL govCanadaURL = getClass().getClassLoader().getResource("GovCanada.gif");
PackageItem govCanadaFile = new PackageItem(govCanadaGif, govCanadaURL);
log.debug(govCanadaFile);

String dir2Path = "some/empty/path/";
PackageItem dir2 = new PackageItem(dir2Path);
log.debug(dir2);
String another = "another/";
PackageItem anotherDir = new PackageItem(another);
log.debug(anotherDir);

String file1Path = "some/path/GovCanada.gif";
URL file1URL = getClass().getClassLoader().getResource("GovCanada.gif");
PackageItem file1 = new PackageItem(file1Path, file1URL);
log.debug(file1);
String anotherPath = "another/path/";
PackageItem anotherPathDir = new PackageItem(anotherPath);
log.debug(anotherPathDir);

String file2Path = "another/path/SymbolCanada.gif";
URL file2URL = getClass().getClassLoader().getResource("SymbolCanada.gif");
PackageItem file2 = new PackageItem(file2Path, file2URL);
log.debug(file2);
String symbolCanadaGif = "another/path/SymbolCanada.gif";
URL symbolCanadaURL = getClass().getClassLoader().getResource("SymbolCanada.gif");
PackageItem symbolCanadaFile = new PackageItem(symbolCanadaGif, symbolCanadaURL);
log.debug(symbolCanadaFile);

String link1Path = "some/path/link2SymbolCanada.gif";
PackageItem link1 = new PackageItem(link1Path, file2Path);
log.debug(link1);
String linkPath = "some/path/link2SymbolCanada.gif";
PackageItem link = new PackageItem(linkPath, "../../another/path/SymbolCanada.gif");
log.debug(link);

List<PackageItem> packageItems = new ArrayList<>();
packageItems.add(dir1);
packageItems.add(dir2);
packageItems.add(file1);
packageItems.add(file2);
packageItems.add(link1);
packageItems.add(someDir);
packageItems.add(somePathDir);
packageItems.add(someEmptyPathDir);
packageItems.add(govCanadaFile);
packageItems.add(anotherDir);
packageItems.add(anotherPathDir);
packageItems.add(symbolCanadaFile);
packageItems.add(link);

String fooPath = "foo";
PackageItem foo = new PackageItem(fooPath);
packageItems.add(foo);

return packageItems;
}
Expand Down

0 comments on commit b8596ab

Please sign in to comment.