Write Properties files in a reproducible way

p2-maven-plugin/src/main/java/org/eclipse/tycho/p2maven/transport/SharedHttpCacheStorage.java

@@ -40,6 +40,7 @@
 import org.codehaus.plexus.component.annotations.Requirement;
 import org.codehaus.plexus.logging.Logger;
 import org.eclipse.equinox.internal.p2.repository.AuthenticationFailedException;
+import org.eclipse.tycho.ReproducibleUtils;
 
 @Component(role = HttpCache.class)
 public class SharedHttpCacheStorage implements HttpCache {
@@ -395,7 +396,7 @@ protected void updateHeader(Headers response, int code) throws IOException, File
 			try (OutputStream out = new BufferedOutputStream(new FileOutputStream(headerFile))) {
 				// we store the header here, this might be a 404 response or (permanent)
 				// redirect we probably need to work with later on
-				header.store(out, null);
+				ReproducibleUtils.storeProperties(header, out, null);
 			}
 		}
 

sisu-osgi/sisu-equinox-launching/src/main/java/org/eclipse/sisu/equinox/launching/internal/DefaultEquinoxInstallationFactory.java

@@ -45,6 +45,7 @@
 import org.eclipse.sisu.equinox.launching.EquinoxInstallation;
 import org.eclipse.sisu.equinox.launching.EquinoxInstallationDescription;
 import org.eclipse.sisu.equinox.launching.EquinoxInstallationFactory;
+import org.eclipse.tycho.ReproducibleUtils;
 import org.eclipse.tycho.TychoConstants;
 import org.osgi.framework.Constants;
 
@@ -151,7 +152,7 @@ public EquinoxInstallation createInstallation(EquinoxInstallationDescription des
             File configurationLocation = configIni.getParentFile();
             configurationLocation.mkdirs();
             try (OutputStream fos = new BufferedOutputStream(new FileOutputStream(configIni))) {
-                p.store(fos, null);
+                ReproducibleUtils.storeProperties(p, fos, null);
             }
 
             return new DefaultEquinoxInstallation(description, location, configurationLocation);
@@ -211,7 +212,7 @@ private String createDevProperties(File location, Map<String, String> devEntries
         Properties properties = new Properties();
         properties.putAll(devEntries);
         try (OutputStream os = new BufferedOutputStream(new FileOutputStream(file))) {
-            properties.store(os, null);
+            ReproducibleUtils.storeProperties(properties, os, null);
         }
         return file.toURI().toURL().toExternalForm();
     }

tycho-api/src/main/java/org/eclipse/tycho/ReproducibleUtils.java

@@ -0,0 +1,54 @@
+/*******************************************************************************
+ * This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License 2.0
+ * which accompanies this distribution, and is available at
+ * https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *******************************************************************************/
+package org.eclipse.tycho;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.Properties;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * Utility methods for reproducible builds.
+ */
+public class ReproducibleUtils {
+    private ReproducibleUtils() {
+    }
+
+    /**
+     * Writes the property list to the output stream in a reproducible way. The java.util.Properties
+     * class writes the lines in a non-reproducible order, adds a non-reproducible timestamp and
+     * uses platform-dependent new line characters.
+     * 
+     * @param properties
+     *            the properties object to write to the output stream.
+     * @param out
+     *            an output stream.
+     * @param comments
+     *            a description of the property list.
+     * @throws IOException
+     *             if writing the property list to the specified output stream throws an
+     *             IOException.
+     */
+    public static void storeProperties(Properties properties, OutputStream out, String comments) throws IOException {
+        final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        properties.store(baos, comments);
+        final String originalContent = baos.toString(StandardCharsets.ISO_8859_1);
+        // Keep the comment lines if any
+        final long commentLinesNb = (comments != null) ? comments.lines().count() : 0;
+        final Stream<String> commentLinesStream = originalContent.lines().limit(commentLinesNb);
+        // Drop the timestamp comment, order the lines and use a system-independent new line
+        final String contentFixed = Stream
+                .concat(commentLinesStream, originalContent.lines().skip(commentLinesNb + 1).sorted())
+                .collect(Collectors.joining("\n", "", "\n"));
+        out.write(contentFixed.getBytes(StandardCharsets.ISO_8859_1));
+    }
+}

tycho-core/src/main/java/org/eclipse/tycho/p2/repository/module/ModuleArtifactMap.java

@@ -29,6 +29,7 @@
 import org.eclipse.core.runtime.IStatus;
 import org.eclipse.core.runtime.Status;
 import org.eclipse.equinox.p2.core.ProvisionException;
+import org.eclipse.tycho.ReproducibleUtils;
 import org.eclipse.tycho.TychoConstants;
 import org.eclipse.tycho.p2.repository.MavenRepositoryCoordinates;
 import org.eclipse.tycho.p2.repository.RepositoryReader;
@@ -163,7 +164,7 @@ private void store() throws ProvisionException {
 
     private static void writeProperties(Properties properties, File outputFile) throws IOException {
         try (OutputStream outputStream = new BufferedOutputStream(new FileOutputStream(outputFile))) {
-            properties.store(outputStream, null);
+            ReproducibleUtils.storeProperties(properties, outputStream, null);
         }
     }
 }

tycho-core/src/main/java/org/eclipse/tycho/p2resolver/P2GeneratorImpl.java

@@ -69,6 +69,7 @@
 import org.eclipse.tycho.OptionalResolutionAction;
 import org.eclipse.tycho.PackagingType;
 import org.eclipse.tycho.ReactorProject;
+import org.eclipse.tycho.ReproducibleUtils;
 import org.eclipse.tycho.TargetEnvironment;
 import org.eclipse.tycho.TychoConstants;
 import org.eclipse.tycho.core.osgitools.BundleReader;
@@ -488,7 +489,7 @@ static void writeArtifactLocations(File outputFile, Map<String, File> artifactLo
 
     private static void writeProperties(Properties properties, File outputFile) throws IOException {
         try (OutputStream outputStream = new BufferedOutputStream(new FileOutputStream(outputFile))) {
-            properties.store(outputStream, null);
+            ReproducibleUtils.storeProperties(properties, outputStream, null);
         }
     }
 

tycho-core/src/main/java/org/eclipse/tycho/p2tools/MirrorApplicationServiceImpl.java

@@ -72,6 +72,7 @@
 import org.eclipse.tycho.ArtifactType;
 import org.eclipse.tycho.BuildDirectory;
 import org.eclipse.tycho.DependencySeed;
+import org.eclipse.tycho.ReproducibleUtils;
 import org.eclipse.tycho.core.shared.StatusTool;
 import org.eclipse.tycho.p2.repository.GAV;
 import org.eclipse.tycho.p2.repository.RepositoryLayoutHelper;
@@ -525,7 +526,7 @@ private void writeP2Index(File repositoryDestination) throws FacadeException {
         properties.setProperty("metadata.repository.factory.order", "content.xml,!");
         try (OutputStream stream = new BufferedOutputStream(
                 new FileOutputStream(new File(repositoryDestination, P2_INDEX_FILE)))) {
-            properties.store(stream, null);
+            ReproducibleUtils.storeProperties(properties, stream, null);
         } catch (IOException e) {
             throw new FacadeException("writing index file failed", e);
         }

tycho-its/projects/resolver.split/org.eclipse.equinox.security/src/org/eclipse/equinox/internal/security/storage/SecurePreferencesRoot.java

@@ -30,6 +30,7 @@
 import org.eclipse.equinox.security.storage.provider.*;
 import org.eclipse.osgi.framework.log.FrameworkLogEntry;
 import org.eclipse.osgi.util.NLS;
+import org.eclipse.tycho.ReproduciblePropertiesUtils;
 
 /**
  * Root secure preference node. In addition to usual things it stores location, modified
@@ -196,7 +197,7 @@ else if (!response.booleanValue())
 		OutputStream stream = null;
 		try {
 			stream = StorageUtils.getOutputStream(location);
-			properties.store(stream, description);
+			ReproduciblePropertiesUtils.store(properties, stream, description);
 			modified = false;
 		} finally {
 			if (stream != null)

tycho-its/src/test/java/org/eclipse/tycho/test/reproducible/ReproducibleBuildTest.java

@@ -9,6 +9,7 @@
 package org.eclipse.tycho.test.reproducible;
 
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.FileSystem;
 import java.nio.file.FileSystems;
 import java.nio.file.Files;
@@ -25,26 +26,40 @@
 import org.junit.Assert;
 import org.junit.Test;
 
+/**
+ * Tests that the build artifacts produced by Tycho are reproducible.
+ */
 public class ReproducibleBuildTest extends AbstractTychoIntegrationTest {
 	// The ZipEntry.getLastModifiedTime() method uses the default timezone to
 	// convert date and time fields to Instant, so we also use the default timezone
 	// for the expected timestamp here.
 	private static final String EXPECTED_TIMESTAMP_STRING = "2023-01-01T00:00:00";
 	private static final Instant EXPECTED_TIMESTAMP_INSTANT = LocalDateTime.parse(EXPECTED_TIMESTAMP_STRING)
 			.toInstant(OffsetDateTime.now().getOffset());
+	Verifier verifier;
 
 	/**
-	 * Check that the build is reproducible.
+	 * Run the maven integration tests related to reproducible builds.
+	 * 
+	 * @throws Exception
 	 */
 	@Test
-	public void test() throws Exception {
-		Verifier verifier = getVerifier("reproducible-build");
+	public void testReproducible() throws Exception {
+		verifier = getVerifier("reproducible-build");
 		verifier.executeGoals(List.of("clean", "verify"));
 		verifier.verifyErrorFreeLog();
 
-		// Check that the timestamp of the files inside the produced archives is equal
-		// to the one specified in the "project.build.outputTimestamp" property of the
-		// pom file.
+		checkArchiveTimestamps();
+		testBuildQualifier();
+		testPropertiesFiles();
+	}
+
+	/**
+	 * Checks that the timestamp of the files inside the produced archives is equal
+	 * to the one specified in the "project.build.outputTimestamp" property of the
+	 * pom file.
+	 */
+	private void checkArchiveTimestamps() throws Exception {
 		checkTimestamps(verifier.getBasedir() + "/reproducible.bundle/target/reproducible.bundle-1.0.0.jar");
 		checkTimestamps(verifier.getBasedir() + "/reproducible.bundle/target/reproducible.bundle-1.0.0-attached.jar");
 		checkTimestamps(verifier.getBasedir() + "/reproducible.bundle/target/reproducible.bundle-1.0.0-sources.jar");
@@ -55,11 +70,8 @@ public void test() throws Exception {
 		checkTimestamps(verifier.getBasedir() + "/reproducible.iu/target/reproducible.iu-1.0.0.zip");
 		checkTimestamps(verifier.getBasedir() + "/reproducible.repository/target/reproducible.repository-1.0.0.zip");
 		checkTimestamps(verifier.getBasedir() + "/reproducible.repository/target/p2-site.zip");
-
-		// Check that the build qualifier uses the timestamp specified in the
-		// "project.build.outputTimestamp" property of the pom file.
-		checkBuildQualifier(verifier.getBasedir()
-				+ "/reproducible.buildqualifier/target/reproducible.buildqualifier-1.0.0-SNAPSHOT.jar");
+		checkTimestamps(
+				verifier.getBasedir() + "/reproducible.repository/target/products/main.product.id-linux.gtk.x86.zip");
 	}
 
 	private void checkTimestamps(String file) throws IOException {
@@ -72,11 +84,34 @@ private void checkTimestamps(String file) throws IOException {
 		}
 	}
 
-	private void checkBuildQualifier(String file) throws IOException {
+	/**
+	 * Checks that the build qualifier uses the timestamp specified in the
+	 * "project.build.outputTimestamp" property of the pom file.
+	 * 
+	 * @throws IOException
+	 */
+	private void testBuildQualifier() throws IOException {
+		final String file = verifier.getBasedir()
+				+ "/reproducible.buildqualifier/target/reproducible.buildqualifier-1.0.0-SNAPSHOT.jar";
 		try (FileSystem fileSystem = FileSystems.newFileSystem(Path.of(file))) {
-			Path manifest = fileSystem.getPath("META-INF/MANIFEST.MF");
-			List<String> lines = Files.readAllLines(manifest);
+			final Path manifest = fileSystem.getPath("META-INF/MANIFEST.MF");
+			final List<String> lines = Files.readAllLines(manifest);
 			Assert.assertTrue(lines.stream().anyMatch(l -> l.equals("Bundle-Version: 1.0.0.202301010000")));
 		}
 	}
+
+	/**
+	 * Checks that the generated properties files are reproducible.
+	 * 
+	 * @throws IOException
+	 */
+	private void testPropertiesFiles() throws IOException {
+		final String file = verifier.getBasedir() + "/reproducible.bundle/target/reproducible.bundle-1.0.0-sources.jar";
+		try (FileSystem fileSystem = FileSystems.newFileSystem(Path.of(file))) {
+			final Path propFile = fileSystem.getPath("OSGI-INF/l10n/bundle-src.properties");
+			final String content = Files.readString(propFile, StandardCharsets.ISO_8859_1);
+			Assert.assertEquals("#Source Bundle Localization\n" + "bundleName=Reproducible-bundle Source\n"
+					+ "bundleVendor=unknown\n", content);
+		}
+	}
 }

tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/MavenP2SiteMojo.java

@@ -72,6 +72,7 @@
 import org.eclipse.equinox.p2.core.IProvisioningAgent;
 import org.eclipse.equinox.p2.repository.artifact.IArtifactRepositoryManager;
 import org.eclipse.tycho.PackagingType;
+import org.eclipse.tycho.ReproducibleUtils;
 import org.eclipse.tycho.TychoConstants;
 import org.eclipse.tycho.core.PGPService;
 import org.eclipse.tycho.p2maven.tools.TychoFeaturesAndBundlesPublisherApplication;
@@ -485,7 +486,7 @@ protected File createMavenAdvice(Artifact artifact) throws MojoExecutionExceptio
             addProvidesAndProperty(properties, TychoConstants.PROP_CLASSIFIER, artifact.getClassifier(), cnt++);
             addProvidesAndProperty(properties, "maven-scope", artifact.getScope(), cnt++);
             try (OutputStream os = new BufferedOutputStream(new FileOutputStream(p2))) {
-                properties.store(os, null);
+                ReproducibleUtils.storeProperties(properties, os, null);
             }
             return p2;
         } catch (IOException e) {

tycho-packaging-plugin/src/main/java/org/eclipse/tycho/packaging/reverseresolve/MavenCentralArtifactCoordinateResolver.java

@@ -36,6 +36,7 @@
 import org.codehaus.plexus.component.annotations.Component;
 import org.codehaus.plexus.component.annotations.Requirement;
 import org.codehaus.plexus.logging.Logger;
+import org.eclipse.tycho.ReproducibleUtils;
 import org.eclipse.tycho.core.shared.MavenContext;
 import org.eclipse.tycho.p2.repository.GAV;
 import org.eclipse.tycho.p2.repository.RepositoryLayoutHelper;
@@ -161,7 +162,7 @@ private void cacheResult(File cacheFile, Dependency dependency) {
 			properties.setProperty(KEY_VERSION, dependency.getVersion());
 			properties.setProperty(KEY_TYPE, dependency.getType());
 			try (OutputStream stream = new BufferedOutputStream(new FileOutputStream(cacheFile))) {
-				properties.store(stream, null);
+				ReproducibleUtils.storeProperties(properties, stream, null);
 			}
 		} catch (IOException e) {
 			// can't create cache file then...

tycho-source-plugin/src/main/java/org/eclipse/tycho/source/OsgiSourceMojo.java

@@ -58,6 +58,7 @@
 import org.eclipse.tycho.BuildPropertiesParser;
 import org.eclipse.tycho.PackagingType;
 import org.eclipse.tycho.ReactorProject;
+import org.eclipse.tycho.ReproducibleUtils;
 import org.eclipse.tycho.TychoProperties;
 import org.eclipse.tycho.core.TychoProject;
 import org.eclipse.tycho.core.osgitools.BundleReader;
@@ -278,7 +279,7 @@ static Resource generateL10nFile(MavenProject project, Path basedir, UnaryOperat
         File l10nPropsFile = new File(l10nOutputDir, MANIFEST_BUNDLE_LOCALIZATION_FILENAME);
         l10nPropsFile.getParentFile().mkdirs();
         try (OutputStream out = new BufferedOutputStream(new FileOutputStream(l10nPropsFile))) {
-            sourceL10nProps.store(out, "Source Bundle Localization");
+            ReproducibleUtils.storeProperties(sourceL10nProps, out, "Source Bundle Localization");
         } catch (IOException e) {
             throw new MojoExecutionException("error while generating source bundle localization file", e);
         }

tycho-source-plugin/src/main/java/org/eclipse/tycho/source/SourceFeatureMojo.java

@@ -54,6 +54,7 @@
 import org.eclipse.tycho.BuildProperties;
 import org.eclipse.tycho.BuildPropertiesParser;
 import org.eclipse.tycho.PackagingType;
+import org.eclipse.tycho.ReproducibleUtils;
 import org.eclipse.tycho.TargetEnvironment;
 import org.eclipse.tycho.TargetPlatform;
 import org.eclipse.tycho.TychoConstants;
@@ -362,7 +363,7 @@ private static Properties readPropertiesIfExists(File propertiesFile) throws IOE
     private static void writeProperties(Properties props, File propertiesFile) throws IOException {
         propertiesFile.getParentFile().mkdirs();
         try (OutputStream out = new BufferedOutputStream(new FileOutputStream(propertiesFile))) {
-            props.store(out, "");
+            ReproducibleUtils.storeProperties(props, out, "");
         }
     }
 

tycho-surefire/tycho-surefire-plugin/src/main/java/org/eclipse/tycho/surefire/AbstractEclipseTestMojo.java

@@ -81,6 +81,7 @@
 import org.eclipse.tycho.OptionalResolutionAction;
 import org.eclipse.tycho.PlatformPropertiesUtils;
 import org.eclipse.tycho.ReactorProject;
+import org.eclipse.tycho.ReproducibleUtils;
 import org.eclipse.tycho.TargetEnvironment;
 import org.eclipse.tycho.TychoConstants;
 import org.eclipse.tycho.core.BundleProject;
@@ -968,7 +969,7 @@ private void storeProperties(Map<String, String> propertiesMap, File file) throw
         p.putAll(propertiesMap);
         try {
             try (OutputStream out = new BufferedOutputStream(new FileOutputStream(file))) {
-                p.store(out, null);
+                ReproducibleUtils.storeProperties(p, out, null);
             }
         } catch (IOException e) {
             throw new MojoExecutionException("Can't write test launcher properties file", e);

tycho-surefire/tycho-surefire-plugin/src/main/java/org/eclipse/tycho/surefire/BndTestMojo.java

@@ -49,6 +49,7 @@
 import org.eclipse.tycho.IllegalArtifactReferenceException;
 import org.eclipse.tycho.MavenArtifactKey;
 import org.eclipse.tycho.PackagingType;
+import org.eclipse.tycho.ReproducibleUtils;
 import org.eclipse.tycho.ResolvedArtifactKey;
 import org.eclipse.tycho.TargetPlatform;
 import org.eclipse.tycho.TychoConstants;
@@ -233,7 +234,7 @@ protected void runTests(ScanResult scanResult) throws MojoExecutionException, Mo
         properties.setProperty(Constants.RUNPROPERTIES, buildRunProperties());
         try {
             try (OutputStream out = new BufferedOutputStream(new FileOutputStream(runfile))) {
-                properties.store(out, null);
+                ReproducibleUtils.storeProperties(properties, out, null);
             }
             String javaExecutable = getJavaExecutable();
             int returncode = container.execute(runfile, "testing", work, (file, bndrun, run) -> {