feat: implement --skip-existing flag (prefix-dev#743)

src/build.rs

@@ -1,29 +1,63 @@
 //! The build module contains the code for running the build process for a given [`Output`]
 use fs_err as fs;
+use rattler_conda_types::{MatchSpec, ParseStrictness};
 use std::path::PathBuf;
+use std::vec;
 
 use miette::IntoDiagnostic;
 use rattler_index::index;
 
 use crate::metadata::Output;
 use crate::package_test::TestConfiguration;
 use crate::recipe::parser::TestType;
+use crate::render::solver::load_repodatas;
 use crate::{package_test, tool_configuration};
 
+/// Check if the build should be skipped because it already exists in any of the channels
+pub async fn skip_existing(
+    output: &Output,
+    tool_configuration: &tool_configuration::Configuration,
+) -> miette::Result<bool> {
+    // If we should skip existing builds, check if the build already exists
+    if tool_configuration.skip_existing {
+        let channels = output.reindex_channels().into_diagnostic()?;
+        let match_spec =
+            MatchSpec::from_str(output.name().as_normalized(), ParseStrictness::Strict)
+                .into_diagnostic()?;
+        let match_spec_vec = vec![match_spec.clone()];
+        let (_, existing) = load_repodatas(
+            &channels,
+            output.target_platform(),
+            tool_configuration,
+            &match_spec_vec,
+        )
+        .await
+        .unwrap();
+
+        return Ok(existing.iter().flatten().any(|package| {
+            package.package_record.version.to_string() == output.version()
+                && output.build_string() == Some(&package.package_record.build)
+        }));
+    }
+    Ok(false)
+}
+
 /// Run the build for the given output. This will fetch the sources, resolve the dependencies,
 /// and execute the build script. Returns the path to the resulting package.
 pub async fn run_build(
     output: Output,
     tool_configuration: &tool_configuration::Configuration,
 ) -> miette::Result<(Output, PathBuf)> {
+    if output.build_string().is_none() {
+        miette::bail!("Build string is not set for {:?}", output.name());
+    }
+
     output
         .build_configuration
         .directories
         .create_build_dir()
         .into_diagnostic()?;
-    if output.build_string().is_none() {
-        miette::bail!("Build string is not set for {:?}", output.name());
-    }
+
     let span = tracing::info_span!("Running build for", recipe = output.identifier().unwrap());
     let _enter = span.enter();
     output.record_build_start();

src/lib.rs

@@ -32,6 +32,7 @@ mod unix;
 pub mod upload;
 mod windows;
 
+use build::skip_existing;
 use dunce::canonicalize;
 use fs_err as fs;
 use metadata::Output;
@@ -123,6 +124,7 @@ pub fn get_tool_config(
         use_zstd: args.common.use_zstd,
         use_bz2: args.common.use_bz2,
         render_only: args.render_only,
+        skip_existing: args.skip_existing,
     }
 }
 
@@ -305,6 +307,13 @@ pub async fn run_build_from_args(
 ) -> miette::Result<()> {
     let mut outputs: Vec<metadata::Output> = Vec::new();
     for output in build_output {
+        if skip_existing(&output, &tool_config).await? {
+            tracing::info!(
+                "Skipping build for {:?}",
+                output.identifier().unwrap_or_else(|| "unknown".to_string())
+            );
+            continue;
+        }
         let output = match run_build(output, &tool_config).await {
             Ok((output, _archive)) => {
                 output.record_build_end();
@@ -416,6 +425,7 @@ pub async fn rebuild_from_args(
         use_zstd: args.common.use_zstd,
         use_bz2: args.common.use_bz2,
         render_only: false,
+        skip_existing: false,
     };
 
     output
@@ -571,7 +581,7 @@ pub fn sort_build_outputs_topologically(
         .iter()
         .map(|idx| &outputs[idx.index()])
         .for_each(|output| {
-            tracing::debug!("ordered output: {:?}", output.name().as_normalized());
+            tracing::debug!("Ordered output: {:?}", output.name().as_normalized());
         });
 
     // Reorder outputs based on the sorted indices

src/opt.rs

@@ -253,6 +253,10 @@ pub struct BuildOpts {
     /// Launch the terminal user interface.
     #[arg(long, default_value = "false", hide = !cfg!(feature = "tui"))]
     pub tui: bool,
+
+    /// Wether to skip packages that already exist in any channel
+    #[arg(long, default_value = "false")]
+    pub skip_existing: bool,
 }
 
 /// Test options.

src/render/solver.rs

@@ -71,13 +71,9 @@ pub async fn create_environment(
     channels: &[String],
     tool_configuration: &tool_configuration::Configuration,
 ) -> anyhow::Result<Vec<RepoDataRecord>> {
-    let channel_config = ChannelConfig::default();
     // Parse the specs from the command line. We do this explicitly instead of allow clap to deal
     // with this because we need to parse the `channel_config` when parsing matchspecs.
 
-    // Find the default cache directory. Create it if it doesn't exist yet.
-    let cache_dir = rattler::default_cache_dir()?;
-
     tracing::info!("\nResolving environment for:\n");
     tracing::info!("  Platform: {}", target_platform);
     tracing::info!("  Channels: ");
@@ -89,72 +85,12 @@ pub async fn create_environment(
         tracing::info!("   - {}", spec);
     }
 
-    std::fs::create_dir_all(&cache_dir)
-        .map_err(|e| anyhow::anyhow!("could not create cache directory: {}", e))?;
-
-    // Determine the channels to use from the command line or select the default. Like matchspecs
-    // this also requires the use of the `channel_config` so we have to do this manually.
-    let channels = channels
-        .iter()
-        .map(|channel_str| Channel::from_str(channel_str, &channel_config))
-        .collect::<Result<Vec<_>, _>>()?;
-
-    // Each channel contains multiple subdirectories. Users can specify the subdirectories they want
-    // to use when specifying their channels. If the user didn't specify the default subdirectories
-    // we use defaults based on the current platform.
-    let platforms = [Platform::NoArch, *target_platform];
-    let channel_urls = channels
-        .iter()
-        .flat_map(|channel| {
-            platforms
-                .iter()
-                .map(move |platform| (channel.clone(), *platform))
-        })
-        .collect::<Vec<_>>();
-
-    // Determine the packages that are currently installed in the environment.
     let installed_packages = find_installed_packages(target_prefix, 100)
         .await
         .context("failed to determine currently installed packages")?;
 
-    // For each channel/subdirectory combination, download and cache the `repodata.json` that should
-    // be available from the corresponding Url. The code below also displays a nice CLI progress-bar
-    // to give users some more information about what is going on.
-
-    let repodata_cache_path = cache_dir.join("repodata");
-    let channel_and_platform_len = channel_urls.len();
-    let repodata_download_client = tool_configuration.client.clone();
-    let sparse_repo_datas = futures::stream::iter(channel_urls)
-        .map(move |(channel, platform)| {
-            let repodata_cache = repodata_cache_path.clone();
-            let download_client = repodata_download_client.clone();
-            async move {
-                fetch_repo_data_records_with_progress(
-                    channel,
-                    platform,
-                    &repodata_cache,
-                    download_client.clone(),
-                    tool_configuration.fancy_log_handler.clone(),
-                    platform != Platform::NoArch,
-                )
-                .await
-            }
-        })
-        .buffered(channel_and_platform_len)
-        .collect::<Vec<_>>()
-        .await
-        // Collect into another iterator where we extract the first erroneous result
-        .into_iter()
-        .filter_map(Result::transpose)
-        .collect::<Result<Vec<_>, _>>()?;
-
-    // Get the package names from the matchspecs so we can only load the package records that we need.
-    let package_names = specs.iter().filter_map(|spec| spec.name.clone());
-    let repodatas = wrap_in_progress(
-        "parsing repodata",
-        &tool_configuration.fancy_log_handler,
-        move || SparseRepoData::load_records_recursive(&sparse_repo_datas, package_names, None),
-    )??;
+    let (cache_dir, repodatas) =
+        load_repodatas(channels, target_platform, tool_configuration, specs).await?;
 
     // Determine virtual packages of the system. These packages define the capabilities of the
     // system. Some packages depend on these virtual packages to indicate compatibility with the
@@ -211,6 +147,71 @@ pub async fn create_environment(
     Ok(required_packages)
 }
 
+/// Load repodata for given matchspecs and channels.
+pub async fn load_repodatas(
+    channels: &[String],
+    target_platform: &Platform,
+    tool_configuration: &tool_configuration::Configuration,
+    specs: &[MatchSpec],
+) -> Result<(PathBuf, Vec<Vec<RepoDataRecord>>), anyhow::Error> {
+    let channel_config = ChannelConfig::default();
+    let cache_dir = rattler::default_cache_dir()?;
+    std::fs::create_dir_all(&cache_dir)
+        .map_err(|e| anyhow::anyhow!("could not create cache directory: {}", e))?;
+
+    let channels = channels
+        .iter()
+        .map(|channel_str| Channel::from_str(channel_str, &channel_config))
+        .collect::<Result<Vec<_>, _>>()?;
+
+    let platforms = [Platform::NoArch, *target_platform];
+    let channel_urls = channels
+        .iter()
+        .flat_map(|channel| {
+            platforms
+                .iter()
+                .map(move |platform| (channel.clone(), *platform))
+        })
+        .collect::<Vec<_>>();
+
+    let repodata_cache_path = cache_dir.join("repodata");
+
+    let channel_and_platform_len = channel_urls.len();
+    let repodata_download_client = tool_configuration.client.clone();
+    let sparse_repo_datas = futures::stream::iter(channel_urls)
+        .map(move |(channel, platform)| {
+            let repodata_cache = repodata_cache_path.clone();
+            let download_client = repodata_download_client.clone();
+            async move {
+                fetch_repo_data_records_with_progress(
+                    channel,
+                    platform,
+                    &repodata_cache,
+                    download_client.clone(),
+                    tool_configuration.fancy_log_handler.clone(),
+                    platform != Platform::NoArch,
+                )
+                .await
+            }
+        })
+        .buffered(channel_and_platform_len)
+        .collect::<Vec<_>>()
+        .await
+        // Collect into another iterator where we extract the first erroneous result
+        .into_iter()
+        .filter_map(Result::transpose)
+        .collect::<Result<Vec<_>, _>>()?;
+
+    let package_names = specs.iter().filter_map(|spec| spec.name.clone());
+    let repodatas = wrap_in_progress(
+        "parsing repodata",
+        &tool_configuration.fancy_log_handler,
+        move || SparseRepoData::load_records_recursive(&sparse_repo_datas, package_names, None),
+    )??;
+
+    Ok((cache_dir, repodatas))
+}
+
 pub async fn install_packages(
     required_packages: &Vec<RepoDataRecord>,
     target_platform: &Platform,

src/tool_configuration.rs

@@ -33,6 +33,9 @@ pub struct Configuration {
 
     /// Whether to only render the build output
     pub render_only: bool,
+
+    /// Wether to skip existing packages
+    pub skip_existing: bool,
 }
 
 /// Get the authentication storage from the given file
@@ -77,6 +80,7 @@ impl Default for Configuration {
             use_zstd: true,
             use_bz2: true,
             render_only: false,
+            skip_existing: false,
         }
     }
 }