From 29008eebc05c616fb9902ade074815823cf7289d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20L=C3=A4ubrich?= Date: Sat, 28 Dec 2024 18:02:55 +0100 Subject: [PATCH] Add support for reading artifacts from global bundle pools Since a while P2 offers a utility method to get access to the shared bundlepools that allow to reuse artifacts already downloaded. This now adds a first implementation of ArtifactDownloadProvider that make use of this shared pools to provide artifacts from the local pools instead of download them through remote sites. --- ...efaultSimpleArtifactRepositoryFactory.java | 36 +++ .../BundlePoolArtifactDownloadProvider.java | 217 ++++++++++++++++++ src/site/markdown/SystemProperties.md | 3 + .../org/eclipse/tycho/TychoConstants.java | 3 + .../tycho/helper/MavenPropertyHelper.java | 14 +- 5 files changed, 272 insertions(+), 1 deletion(-) create mode 100644 p2-maven-plugin/src/main/java/org/eclipse/tycho/p2maven/repository/DefaultSimpleArtifactRepositoryFactory.java create mode 100644 p2-maven-plugin/src/main/java/org/eclipse/tycho/p2maven/transport/BundlePoolArtifactDownloadProvider.java diff --git a/p2-maven-plugin/src/main/java/org/eclipse/tycho/p2maven/repository/DefaultSimpleArtifactRepositoryFactory.java b/p2-maven-plugin/src/main/java/org/eclipse/tycho/p2maven/repository/DefaultSimpleArtifactRepositoryFactory.java new file mode 100644 index 0000000000..6b29eef6f4 --- /dev/null +++ b/p2-maven-plugin/src/main/java/org/eclipse/tycho/p2maven/repository/DefaultSimpleArtifactRepositoryFactory.java @@ -0,0 +1,36 @@ +/******************************************************************************* + * Copyright (c) 2024 Christoph Läubrich and others. + * This program and the accompanying materials + * are made available under the terms of the Eclipse Public License 2.0 + * which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Christoph Läubrich - initial API and implementation + *******************************************************************************/ +package org.eclipse.tycho.p2maven.repository; + +import javax.inject.Inject; +import javax.inject.Named; + +import org.eclipse.equinox.internal.p2.artifact.repository.simple.SimpleArtifactRepositoryFactory; +import org.eclipse.equinox.p2.core.IProvisioningAgent; + +@Named +public class DefaultSimpleArtifactRepositoryFactory extends SimpleArtifactRepositoryFactory { + + private IProvisioningAgent agent; + + @Inject + public DefaultSimpleArtifactRepositoryFactory(IProvisioningAgent agent) { + this.agent = agent; + } + + @Override + protected IProvisioningAgent getAgent() { + return agent; + } + +} diff --git a/p2-maven-plugin/src/main/java/org/eclipse/tycho/p2maven/transport/BundlePoolArtifactDownloadProvider.java b/p2-maven-plugin/src/main/java/org/eclipse/tycho/p2maven/transport/BundlePoolArtifactDownloadProvider.java new file mode 100644 index 0000000000..2571d8558c --- /dev/null +++ b/p2-maven-plugin/src/main/java/org/eclipse/tycho/p2maven/transport/BundlePoolArtifactDownloadProvider.java @@ -0,0 +1,217 @@ +/******************************************************************************* + * Copyright (c) 2024 Christoph Läubrich and others. + * This program and the accompanying materials + * are made available under the terms of the Eclipse Public License 2.0 + * which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Christoph Läubrich - initial API and implementation + *******************************************************************************/ +package org.eclipse.tycho.p2maven.transport; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.DigestOutputStream; +import java.security.MessageDigest; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Stream; + +import javax.inject.Inject; +import javax.inject.Named; + +import org.codehaus.plexus.logging.Logger; +import org.eclipse.core.runtime.IStatus; +import org.eclipse.core.runtime.Status; +import org.eclipse.equinox.internal.p2.artifact.repository.simple.SimpleArtifactRepositoryFactory; +import org.eclipse.equinox.internal.p2.repository.DownloadStatus; +import org.eclipse.equinox.internal.p2.repository.helpers.ChecksumHelper; +import org.eclipse.equinox.internal.p2.repository.helpers.RepositoryHelper; +import org.eclipse.equinox.p2.core.ProvisionException; +import org.eclipse.equinox.p2.metadata.IArtifactKey; +import org.eclipse.equinox.p2.repository.artifact.IArtifactDescriptor; +import org.eclipse.equinox.p2.repository.artifact.IArtifactRepository; +import org.eclipse.equinox.p2.repository.artifact.IFileArtifactRepository; +import org.eclipse.tycho.TychoConstants; +import org.eclipse.tycho.helper.MavenPropertyHelper; +import org.eclipse.tycho.transport.ArtifactDownloadProvider; + +/** + * Provides artifacts already available from the users bundle pools + */ +@Named +public class BundlePoolArtifactDownloadProvider implements ArtifactDownloadProvider { + + private SimpleArtifactRepositoryFactory artifactRepositoryFactory; + private Map repositoryMap = new ConcurrentHashMap<>(); + private TransportCacheConfig cacheConfig; + private Logger logger; + private boolean useSharedPools; + private boolean useWorkspacePools; + private int priority; + + @Inject + public BundlePoolArtifactDownloadProvider(SimpleArtifactRepositoryFactory artifactRepositoryFactory, + TransportCacheConfig cacheConfig, Logger logger, MavenPropertyHelper propertyHelper) { + this.artifactRepositoryFactory = artifactRepositoryFactory; + this.cacheConfig = cacheConfig; + this.logger = logger; + useSharedPools = propertyHelper.getGlobalBooleanProperty("tycho.p2.transport.bundlepools.shared", true); + useWorkspacePools = propertyHelper.getGlobalBooleanProperty("tycho.p2.transport.bundlepools.workspace", true); + priority = propertyHelper.getGlobalIntProperty("tycho.p2.transport.bundlepools.priority", 100); + + } + + @Override + public IStatus downloadArtifact(URI source, OutputStream target, IArtifactDescriptor originalDescriptor) { + return pools().parallel().flatMap(path -> { + IArtifactRepository repository = getRepository(path); + if (repository instanceof IFileArtifactRepository filerepository) { + IArtifactKey artifactKey = originalDescriptor.getArtifactKey(); + File artifactFile = filerepository.getArtifactFile(artifactKey); + if (artifactFile != null) { + return Arrays.stream(repository.getArtifactDescriptors(artifactKey)).map( + descriptor -> new RepositoryCandidate(filerepository, descriptor, artifactFile.toPath())); + } + } + return Stream.empty(); + }).filter(cand -> isMatch(cand, originalDescriptor)).findAny().map(candidate -> { + IArtifactRepository repository = candidate.repository(); + if (cacheConfig.isInteractive()) { + logger.info("Reading from " + repository.getName() + ": " + candidate.artifactFile()); + } + return copyToTarget(target, candidate.artifactFile()); + }).orElse(Status.CANCEL_STATUS); + } + + /** + * Test if two descriptors have at least one matching hashsum in which case we + * assume they are describing the same artifact and not only have the same + * version/id, this should not happen usually, but as we use global pools here + * it is better to be safe than sorry. + * + * @param candidate the candidate we want to use + * @param originalDescriptor the original descriptor queried + * @return true if at least one hashsum matches in both descriptors + */ + private boolean isMatch(RepositoryCandidate candidate, IArtifactDescriptor originalDescriptor) { + Path artifactFile = candidate.artifactFile(); + if (Files.isRegularFile(artifactFile)) { + // we can only use files as we need to process them as if downloaded from a real + // server... + IArtifactDescriptor repositoryDescriptor = candidate.descriptor(); + // now see if we can perform a fast check by comparing original hashsums + for (Entry entry : originalDescriptor.getProperties().entrySet()) { + String key = entry.getKey(); + if (key.startsWith(TychoConstants.PROP_DOWNLOAD_CHECKSUM_PREFIX)) { + String property = repositoryDescriptor.getProperty(key); + if (property != null) { + String value = entry.getValue(); + return value.equals(property); + } + } + } + if (fileSizeMatch(repositoryDescriptor, originalDescriptor)) { + // if we are here, then it means no download checksums where present for + // comparison and we need to generate one ourself + for (Entry entry : originalDescriptor.getProperties().entrySet()) { + String key = entry.getKey(); + if (key.startsWith(TychoConstants.PROP_DOWNLOAD_CHECKSUM_PREFIX)) { + try { + String algorithm = key.substring(TychoConstants.PROP_DOWNLOAD_CHECKSUM_PREFIX.length()) + .toUpperCase(); + MessageDigest md = MessageDigest.getInstance(algorithm); + try (DigestOutputStream outputStream = new DigestOutputStream( + OutputStream.nullOutputStream(), md); + InputStream inputStream = Files.newInputStream(artifactFile)) { + inputStream.transferTo(outputStream); + } + return ChecksumHelper.toHexString(md.digest()).equals(entry.getValue()); + } catch (Exception e) { + // can't check... + } + } + } + } + } + return false; + } + + private static IStatus copyToTarget(OutputStream target, Path path) { + try { + Files.copy(path, target); + } catch (IOException e) { + return Status.error("Can't copy file to target", e); + } + DownloadStatus status = new DownloadStatus(IStatus.OK, "org.eclipse.tycho", "File " + path, null); + try { + status.setFileSize(Files.size(path)); + } catch (IOException e) { + } + try { + status.setLastModified(Files.getLastModifiedTime(path).toMillis()); + } catch (IOException e) { + } + return status; + } + + private boolean fileSizeMatch(IArtifactDescriptor repositoryDescriptor, IArtifactDescriptor originalDescriptor) { + String originalSize = originalDescriptor.getProperty(IArtifactDescriptor.DOWNLOAD_SIZE); + if (originalSize != null) { + String property = repositoryDescriptor.getProperty(IArtifactDescriptor.DOWNLOAD_SIZE); + if (property != null) { + return originalSize.equals(property); + } + } + // assume true for further processing + return true; + } + + private Stream pools() { + if (useSharedPools) { + if (useWorkspacePools) { + List sharedBundlePools = RepositoryHelper.getSharedBundlePools(); + List workspaceBundlePools = RepositoryHelper.getWorkspaceBundlePools(); + return Stream.concat(sharedBundlePools.stream(), workspaceBundlePools.stream()).distinct(); + } else { + return RepositoryHelper.getSharedBundlePools().stream(); + } + } else if (useWorkspacePools) { + return RepositoryHelper.getWorkspaceBundlePools().stream(); + } else { + return Stream.empty(); + } + } + + private IArtifactRepository getRepository(Path path) { + return repositoryMap.computeIfAbsent(path, p -> { + try { + return artifactRepositoryFactory.load(path.toUri(), 0, null); + } catch (ProvisionException e) { + return null; + } + }); + } + + private static record RepositoryCandidate(IFileArtifactRepository repository, IArtifactDescriptor descriptor, + Path artifactFile) { + + } + + @Override + public int getPriority() { + return priority; + } + +} diff --git a/src/site/markdown/SystemProperties.md b/src/site/markdown/SystemProperties.md index 97371ac5d8..85fcb0ffc3 100644 --- a/src/site/markdown/SystemProperties.md +++ b/src/site/markdown/SystemProperties.md @@ -46,3 +46,6 @@ tycho.p2.transport.cache | file path | local maven repository | Specify the loca tycho.p2.transport.debug | true/false | false | enable debugging of the Tycho Transport tycho.p2.transport.max-download-threads | number | 4 | maximum number of threads that should be used to download artifacts in parallel tycho.p2.transport.min-cache-minutes | number | 60 | Number of minutes that a cache entry is assumed to be fresh and is not fetched again from the server +tycho.p2.transport.bundlepools.priority | number | 100 | priority used for bundle pools +tycho.p2.transport.bundlepools.shared | true/false | true | query shared bundle pools for artifacts before downloading them from remote servers +tycho.p2.transport.bundlepools.workspace | true/false | true | query Workspace bundle pools for artifacts before downloading them from remote servers diff --git a/tycho-api/src/main/java/org/eclipse/tycho/TychoConstants.java b/tycho-api/src/main/java/org/eclipse/tycho/TychoConstants.java index 4595fac7a6..bb40fde78b 100644 --- a/tycho-api/src/main/java/org/eclipse/tycho/TychoConstants.java +++ b/tycho-api/src/main/java/org/eclipse/tycho/TychoConstants.java @@ -17,6 +17,8 @@ import java.io.File; import java.util.regex.Pattern; +import org.eclipse.equinox.p2.repository.artifact.IArtifactDescriptor; + public interface TychoConstants { String USER_HOME = System.getProperty("user.home"); @@ -152,4 +154,5 @@ public interface TychoConstants { String SUFFIX_QUALIFIER = ".qualifier"; String SUFFIX_SNAPSHOT = "-SNAPSHOT"; + String PROP_DOWNLOAD_CHECKSUM_PREFIX = IArtifactDescriptor.DOWNLOAD_CHECKSUM + "."; } diff --git a/tycho-spi/src/main/java/org/eclipse/tycho/helper/MavenPropertyHelper.java b/tycho-spi/src/main/java/org/eclipse/tycho/helper/MavenPropertyHelper.java index dbd6db4847..480f01d409 100644 --- a/tycho-spi/src/main/java/org/eclipse/tycho/helper/MavenPropertyHelper.java +++ b/tycho-spi/src/main/java/org/eclipse/tycho/helper/MavenPropertyHelper.java @@ -90,7 +90,19 @@ public String getGlobalProperty(String key, String defaultValue) { return systemProperty; } } - // java sysem properties last + // java system properties last return System.getProperty(key, defaultValue); } + + public boolean getGlobalBooleanProperty(String key, boolean defaultValue) { + return Boolean.parseBoolean(getGlobalProperty(key, Boolean.toString(defaultValue))); + } + + public int getGlobalIntProperty(String key, int defaultValue) { + try { + return Integer.parseInt(getGlobalProperty(key, Integer.toString(defaultValue))); + } catch (NumberFormatException e) { + return defaultValue; + } + } }