Skip to content

Commit

Permalink
Fix checksum validation for archive generation download
Browse files Browse the repository at this point in the history
  • Loading branch information
@yma96
yma96 committed Dec 2, 2024
1 parent 95850b5 commit 1ffe209
Show file tree
Hide file tree
Showing 3 changed files with 122 additions and 49 deletions.
90 changes: 79 additions & 11 deletions src/main/java/org/commonjava/indy/service/archive/controller/ArchiveController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import org.commonjava.indy.service.archive.config.PreSeedConfig;
import org.commonjava.indy.service.archive.model.ArchiveStatus;
import org.commonjava.indy.service.archive.model.dto.HistoricalContentDTO;
import org.commonjava.indy.service.archive.model.dto.HistoricalEntryDTO;
import org.commonjava.indy.service.archive.util.HistoricalContentListReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand All @@ -47,10 +48,15 @@
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorCompletionService;
Expand Down Expand Up @@ -79,6 +85,15 @@ public class ArchiveController

private final String PART_ARCHIVE_SUFFIX = PART_SUFFIX + ARCHIVE_SUFFIX;

private static final Set<String> CHECKSUMS = Collections.unmodifiableSet( new HashSet<String>()
{
{
add( ".md5" );
add( ".sha1" );
add( ".sha256" );
}
} );

@Inject
HistoricalContentListReader reader;

Expand Down Expand Up @@ -146,11 +161,14 @@ protected Boolean doGenerate( HistoricalContentDTO content )
content.getBuildConfigId() );
recordInProgress( content.getBuildConfigId() );

Map<String, String> downloadPaths = reader.readPaths( content );
Map<String, HistoricalEntryDTO> entryDTOs = reader.readEntries( content );
Map<String, String> downloadPaths = new HashMap<>();
entryDTOs.forEach( ( key, value ) -> downloadPaths.put( key, value.getPath() ) );

Optional<File> archive;
try
{
downloadArtifacts( downloadPaths, content );
downloadArtifacts( entryDTOs, downloadPaths, content );
archive = generateArchive( new ArrayList<>( downloadPaths.values() ), content );
}
catch ( final InterruptedException e )
Expand Down Expand Up @@ -226,7 +244,8 @@ public String getStatus( String buildConfigId )
return treated.get( buildConfigId );
}

private void downloadArtifacts( final Map<String, String> downloadPaths, final HistoricalContentDTO content )
private void downloadArtifacts( final Map<String, HistoricalEntryDTO> entryDTOs,
final Map<String, String> downloadPaths, final HistoricalContentDTO content )
throws InterruptedException, ExecutionException, IOException
{
BasicCookieStore cookieStore = new BasicCookieStore();
Expand All @@ -236,13 +255,23 @@ private void downloadArtifacts( final Map<String, String> downloadPaths, final H
File dir = new File( contentBuildDir );
dir.delete();

fileTrackedContent( contentBuildDir, content );
unpackHistoricalArchive( contentBuildDir, content.getBuildConfigId() );
HistoricalContentDTO originalTracked = unpackHistoricalArchive( contentBuildDir, content.getBuildConfigId() );
Map<String, List<String>> originalChecksumsMap = new HashMap<>();
if ( originalTracked != null )
{
Map<String, HistoricalEntryDTO> originalEntries = reader.readEntries( originalTracked );
originalEntries.forEach( ( key, entry ) -> originalChecksumsMap.put( key, new ArrayList<>(
Arrays.asList( entry.getSha1(), entry.getSha256(), entry.getMd5() ) ) ) );
}

for ( String path : downloadPaths.keySet() )
{
String filePath = downloadPaths.get( path );
executor.submit( download( contentBuildDir, path, filePath, cookieStore ) );
HistoricalEntryDTO entry = entryDTOs.get( path );
List<String> checksums =
new ArrayList<>( Arrays.asList( entry.getSha1(), entry.getSha256(), entry.getMd5() ) );
List<String> originalChecksums = originalChecksumsMap.get( path );
executor.submit( download( contentBuildDir, path, filePath, checksums, originalChecksums, cookieStore ) );
}
int success = 0;
int failed = 0;
Expand All @@ -257,6 +286,8 @@ private void downloadArtifacts( final Map<String, String> downloadPaths, final H
failed++;
}
}
// file the latest tracked json at the end
fileTrackedContent( contentBuildDir, content );
logger.info( "Artifacts download completed, success:{}, failed:{}", success, failed );
}

Expand Down Expand Up @@ -369,14 +400,14 @@ private void fileTrackedContent( String contentBuildDir, final HistoricalContent
}
}

private void unpackHistoricalArchive( String contentBuildDir, String buildConfigId )
private HistoricalContentDTO unpackHistoricalArchive( String contentBuildDir, String buildConfigId )
throws IOException
{
final File archive = new File( archiveDir, buildConfigId + ARCHIVE_SUFFIX );
if ( !archive.exists() )
{
logger.debug( "Don't find historical archive for buildConfigId: {}.", buildConfigId );
return;
return null;
}

logger.info( "Start unpacking historical archive for buildConfigId: {}.", buildConfigId );
Expand All @@ -393,16 +424,53 @@ private void unpackHistoricalArchive( String contentBuildDir, String buildConfig

}
inputStream.close();

File originalTracked = new File( contentBuildDir, buildConfigId );
if ( originalTracked.exists() )
{
return objectMapper.readValue( originalTracked, HistoricalContentDTO.class );
}
else
{
logger.debug( "No tracked json file found after zip unpack for buildConfigId {}", buildConfigId );
return null;
}
}

private Callable<Boolean> download( String contentBuildDir, final String path, final String filePath,
private boolean validateChecksum( final String filePath, final List<String> current, final List<String> original )
{
if ( CHECKSUMS.stream().anyMatch( suffix -> filePath.toLowerCase().endsWith( "." + suffix ) ) )
{
// skip to validate checksum files
return false;
}
if ( original == null || original.isEmpty() || original.stream().allMatch( Objects::isNull ) )
{
return false;
}
if ( original.get( 0 ) != null && original.get( 0 ).equals( current.get( 0 ) ) )
{
return true;
}
if ( original.get( 1 ) != null && original.get( 1 ).equals( current.get( 1 ) ) )
{
return true;
}
return original.get( 2 ) != null && original.get( 2 ).equals( current.get( 2 ) );
}

private Callable<Boolean> download( final String contentBuildDir, final String path, final String filePath,
final List<String> checksums, final List<String> originalChecksums,
final CookieStore cookieStore )
{
return () -> {
final File target = new File( contentBuildDir, filePath );
if ( target.exists() )

if ( target.exists() && validateChecksum( filePath, checksums, originalChecksums ) )
{
logger.trace( "<<<Already existed in historical archive, skip downloading, path: {}.", path );
logger.debug(
"<<<Already existed in historical archive, and checksum equals, skip downloading, path: {}.",
path );
return true;
}
final File dir = target.getParentFile();
Expand Down
63 changes: 32 additions & 31 deletions src/main/java/org/commonjava/indy/service/archive/util/HistoricalContentListReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,12 @@
*/
package org.commonjava.indy.service.archive.util;

import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import org.commonjava.indy.service.archive.config.PreSeedConfig;
import org.commonjava.indy.service.archive.model.dto.HistoricalContentDTO;
import org.commonjava.indy.service.archive.model.dto.HistoricalEntryDTO;

import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import java.util.HashMap;
import java.util.Map;

Expand All @@ -43,41 +43,42 @@ public HistoricalContentListReader( PreSeedConfig preSeedConfig )
this.preSeedConfig = preSeedConfig;
}

public Map<String, String> readPaths( HistoricalContentDTO content )
public Map<String, HistoricalEntryDTO> readEntries( HistoricalContentDTO content )
{
Map<String, String> pathMap = new HashMap<>();
Map<String, HistoricalEntryDTO> pathMap = new HashMap<>();
HistoricalEntryDTO[] downloads = content.getDownloads();

if ( downloads != null )
if ( downloads == null )
{
for ( HistoricalEntryDTO download : downloads )
{
String path = download.getPath();
String packageType = download.getStoreKey().getPackageType();
return pathMap;
}
for ( HistoricalEntryDTO download : downloads )
{
String path = download.getPath();
String packageType = download.getStoreKey().getPackageType();

if ( packageType.equals( NPM_PKG_KEY ) && !path.endsWith( ".tgz" ) )
{
// Ignore the npm package metadata in archive
continue;
}
if ( path.contains( "maven-metadata.xml" ) )
{
// Ignore maven-metadata.xml in archive
continue;
}
// ensure every entry has an available localUrl
buildDownloadUrl( download );
if ( packageType.equals( NPM_PKG_KEY ) && !path.endsWith( ".tgz" ) )
{
// Ignore the npm package metadata in archive
continue;
}
if ( path.contains( "maven-metadata.xml" ) )
{
// Ignore maven-metadata.xml in archive
continue;
}
// ensure every entry has an available localUrl
buildDownloadUrl( download );

// local url would be preferred to download artifact
String url = download.getLocalUrl();
if ( url == null )
{
url = download.getOriginUrl();
}
if ( url != null )
{
pathMap.put( url, download.getPath() );
}
// local url would be preferred to download artifact
String url = download.getLocalUrl();
if ( url == null )
{
url = download.getOriginUrl();
}
if ( url != null )
{
pathMap.put( url, download );
}
}
return pathMap;
Expand Down
18 changes: 11 additions & 7 deletions src/test/java/org/commonjava/indy/service/archive/util/HistoricalContentListReaderTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,14 @@
package org.commonjava.indy.service.archive.util;

import io.quarkus.test.junit.QuarkusTest;
import org.commonjava.indy.service.archive.config.PreSeedConfig;
import org.commonjava.indy.service.archive.model.StoreKey;
import org.commonjava.indy.service.archive.model.StoreType;
import org.commonjava.indy.service.archive.model.dto.HistoricalContentDTO;
import org.commonjava.indy.service.archive.model.dto.HistoricalEntryDTO;
import org.junit.jupiter.api.Test;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
Expand Down Expand Up @@ -58,13 +58,15 @@ public void testMavenReadPaths()
entryDTOs.add( entry );
entryDTOs.add( metaEntry );

HistoricalContentDTO contentDTO = new HistoricalContentDTO( "8888", entryDTOs.toArray(
new HistoricalEntryDTO[entryDTOs.size()] ) );
HistoricalContentDTO contentDTO =
new HistoricalContentDTO( "8888", entryDTOs.toArray( new HistoricalEntryDTO[entryDTOs.size()] ) );

TestPreSeedConfig preSeedConfig = new TestPreSeedConfig( Optional.of( MAIN_INDY ) );
HistoricalContentListReader reader = new HistoricalContentListReader( preSeedConfig );

Map<String, String> paths = reader.readPaths( contentDTO );
Map<String, HistoricalEntryDTO> entryMaps = reader.readEntries( contentDTO );
Map<String, String> paths = new HashMap<>();
entryMaps.forEach( ( key, value ) -> paths.put( key, value.getPath() ) );

assertThat( paths.size(), equalTo( 1 ) );
String storePath = MAIN_INDY + "/api/content" + entry.getStorePath();
Expand All @@ -86,13 +88,15 @@ public void testNPMReadPaths()
entryDTOs.add( npmEntry );
entryDTOs.add( npmMetaEntry );

HistoricalContentDTO contentDTO = new HistoricalContentDTO( "8888", entryDTOs.toArray(
new HistoricalEntryDTO[entryDTOs.size()] ) );
HistoricalContentDTO contentDTO =
new HistoricalContentDTO( "8888", entryDTOs.toArray( new HistoricalEntryDTO[entryDTOs.size()] ) );

TestPreSeedConfig preSeedConfig = new TestPreSeedConfig( Optional.of( MAIN_INDY ) );
HistoricalContentListReader reader = new HistoricalContentListReader( preSeedConfig );

Map<String, String> paths = reader.readPaths( contentDTO );
Map<String, HistoricalEntryDTO> entryMaps = reader.readEntries( contentDTO );
Map<String, String> paths = new HashMap<>();
entryMaps.forEach( ( key, value ) -> paths.put( key, value.getPath() ) );

assertThat( paths.size(), equalTo( 1 ) );
String storePath = MAIN_INDY + "/api/content" + npmEntry.getStorePath();
Expand Down

0 comments on commit 1ffe209

Please sign in to comment.