Skip to content

Commit

Permalink
Implementation of shallow libgit2 fetches behind an unstable flag
Browse files Browse the repository at this point in the history
  • Loading branch information
connorworley committed Jan 11, 2024
1 parent 4eef543 commit a1559be
Show file tree
Hide file tree
Showing 10 changed files with 154 additions and 60 deletions.
57 changes: 45 additions & 12 deletions src/cargo/core/features.rs
Original file line number Diff line number Diff line change
Expand Up @@ -770,6 +770,7 @@ unstable_cli_options!(
dual_proc_macros: bool = ("Build proc-macros for both the host and the target"),
features: Option<Vec<String>>,
gc: bool = ("Track cache usage and \"garbage collect\" unused files"),
git: Option<GitFeatures> = ("Enable support for shallow git fetch operations"),
gitoxide: Option<GitoxideFeatures> = ("Use gitoxide for the given git interactions, or all of them if no argument is given"),
host_config: bool = ("Enable the `[host]` section in the .cargo/config.toml file"),
lints: bool = ("Pass `[lints]` to the linting tools"),
Expand Down Expand Up @@ -875,13 +876,47 @@ where
}

#[derive(Debug, Copy, Clone, Default, Deserialize)]
pub struct GitoxideFeatures {
/// All fetches are done with `gitoxide`, which includes git dependencies as well as the crates index.
pub fetch: bool,
pub struct GitFeatures {
/// When cloning the index, perform a shallow clone. Maintain shallowness upon subsequent fetches.
pub shallow_index: bool,
/// When cloning git dependencies, perform a shallow clone and maintain shallowness on subsequent fetches.
pub shallow_deps: bool,
}

impl GitFeatures {
fn all() -> Self {
GitFeatures {
shallow_index: true,
shallow_deps: true,
}
}
}

fn parse_git(it: impl Iterator<Item = impl AsRef<str>>) -> CargoResult<Option<GitFeatures>> {
let mut out = GitFeatures::default();
let GitFeatures {
shallow_index,
shallow_deps,
} = &mut out;

for e in it {
match e.as_ref() {
"shallow-index" => *shallow_index = true,
"shallow-deps" => *shallow_deps = true,
_ => {
bail!(
"unstable 'git' only takes 'shallow-index' and 'shallow-deps' as valid inputs"
)
}
}
}
Ok(Some(out))
}

#[derive(Debug, Copy, Clone, Default, Deserialize)]
pub struct GitoxideFeatures {
/// All fetches are done with `gitoxide`, which includes git dependencies as well as the crates index.
pub fetch: bool,
/// Checkout git dependencies using `gitoxide` (submodules are still handled by git2 ATM, and filters
/// like linefeed conversions are unsupported).
pub checkout: bool,
Expand All @@ -895,9 +930,7 @@ impl GitoxideFeatures {
fn all() -> Self {
GitoxideFeatures {
fetch: true,
shallow_index: true,
checkout: true,
shallow_deps: true,
internal_use_git2: false,
}
}
Expand All @@ -907,9 +940,7 @@ impl GitoxideFeatures {
fn safe() -> Self {
GitoxideFeatures {
fetch: true,
shallow_index: false,
checkout: true,
shallow_deps: false,
internal_use_git2: false,
}
}
Expand All @@ -921,21 +952,17 @@ fn parse_gitoxide(
let mut out = GitoxideFeatures::default();
let GitoxideFeatures {
fetch,
shallow_index,
checkout,
shallow_deps,
internal_use_git2,
} = &mut out;

for e in it {
match e.as_ref() {
"fetch" => *fetch = true,
"shallow-index" => *shallow_index = true,
"shallow-deps" => *shallow_deps = true,
"checkout" => *checkout = true,
"internal-use-git2" => *internal_use_git2 = true,
_ => {
bail!("unstable 'gitoxide' only takes `fetch`, 'shallow-index', 'shallow-deps' and 'checkout' as valid inputs")
bail!("unstable 'gitoxide' only takes `fetch` and 'checkout' as valid inputs")
}
}
}
Expand Down Expand Up @@ -1109,6 +1136,12 @@ impl CliUnstable {
"doctest-xcompile" => self.doctest_xcompile = parse_empty(k, v)?,
"dual-proc-macros" => self.dual_proc_macros = parse_empty(k, v)?,
"gc" => self.gc = parse_empty(k, v)?,
"git" => {
self.git = v.map_or_else(
|| Ok(Some(GitFeatures::all())),
|v| parse_git(v.split(',')),
)?
}
"gitoxide" => {
self.gitoxide = v.map_or_else(
|| Ok(Some(GitoxideFeatures::all())),
Expand Down
11 changes: 6 additions & 5 deletions src/cargo/sources/git/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ mod utils;

/// For `-Zgitoxide` integration.
pub mod fetch {
use crate::core::features::GitoxideFeatures;
use crate::core::features::GitFeatures;
use crate::Config;

/// The kind remote repository to fetch.
Expand All @@ -37,18 +37,19 @@ pub mod fetch {
repo_is_shallow: bool,
config: &Config,
) -> gix::remote::fetch::Shallow {
let has_feature = |cb: &dyn Fn(GitoxideFeatures) -> bool| {
let has_feature = |cb: &dyn Fn(GitFeatures) -> bool| {
config
.cli_unstable()
.gitoxide
.git
.map_or(false, |features| cb(features))
};

// maintain shallow-ness and keep downloading single commits, or see if we can do shallow clones
if !repo_is_shallow {
match self {
RemoteKind::GitDependency if has_feature(&|git| git.shallow_deps) => {}
RemoteKind::Registry if has_feature(&|git| git.shallow_index) => {}
RemoteKind::GitDependency if has_feature(&|features| features.shallow_deps) => {
}
RemoteKind::Registry if has_feature(&|features| features.shallow_index) => {}
_ => return gix::remote::fetch::Shallow::NoChange,
}
};
Expand Down
7 changes: 3 additions & 4 deletions src/cargo/sources/git/oxide.rs
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,9 @@ pub fn with_retry_and_progress(
) -> CargoResult<()> {
std::thread::scope(|s| {
let mut progress_bar = Progress::new("Fetch", config);
let is_shallow = config
.cli_unstable()
.gitoxide
.map_or(false, |gix| gix.shallow_deps || gix.shallow_index);
let is_shallow = config.cli_unstable().git.map_or(false, |features| {
features.shallow_deps || features.shallow_index
});
network::retry::with_retry(config, || {
let progress_root: Arc<gix::progress::tree::Root> =
gix::progress::tree::root::Options {
Expand Down
4 changes: 2 additions & 2 deletions src/cargo/sources/git/source.rs
Original file line number Diff line number Diff line change
Expand Up @@ -108,8 +108,8 @@ impl<'cfg> GitSource<'cfg> {
&source_id,
config
.cli_unstable()
.gitoxide
.map_or(false, |gix| gix.fetch && gix.shallow_deps),
.git
.map_or(false, |features| features.shallow_deps),
);

let source = GitSource {
Expand Down
3 changes: 3 additions & 0 deletions src/cargo/sources/git/utils.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1117,6 +1117,9 @@ pub fn fetch(
if tags {
opts.download_tags(git2::AutotagOption::All);
}
if let gix::remote::fetch::Shallow::DepthAtRemote(depth) = shallow {
opts.depth(0i32.saturating_add_unsigned(depth.get()));
}
// The `fetch` operation here may fail spuriously due to a corrupt
// repository. It could also fail, however, for a whole slew of other
// reasons (aka network related reasons). We want Cargo to automatically
Expand Down
4 changes: 2 additions & 2 deletions src/cargo/sources/registry/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -476,8 +476,8 @@ impl<'cfg> RegistrySource<'cfg> {
source_id,
config
.cli_unstable()
.gitoxide
.map_or(false, |gix| gix.fetch && gix.shallow_index)
.git
.map_or(false, |features| features.shallow_index)
&& !source_id.is_sparse(),
);
let ops = if source_id.is_sparse() {
Expand Down
19 changes: 16 additions & 3 deletions src/doc/src/reference/unstable.md
Original file line number Diff line number Diff line change
Expand Up @@ -1160,19 +1160,32 @@ to run with `gitoxide` with the `-Zgitoxide=operation[,operationN]` syntax.
Valid operations are the following:

* `fetch` - All fetches are done with `gitoxide`, which includes git dependencies as well as the crates index.
* `checkout` *(planned)* - checkout the worktree, with support for filters and submodules.

## git

* Tracking Issue: [TODO]()

With the 'git' unstable feature, both `gitoxide` and `git2` will perform shallow fetches of the crate
index and git dependencies.

While `-Zgit` enables all currently implemented features, one can individually select when to perform
shallow fetches with the `-Zgit=operation[,operationN]` syntax.

Valid operations are the following:

* `shallow-index` - perform a shallow clone of the index.
* `shallow-deps` - perform a shallow clone of git dependencies.
* `checkout` *(planned)* - checkout the worktree, with support for filters and submodules.

**Details on shallow clones**

* To enable shallow clones, add `-Zgitoxide=fetch,shallow_deps` for fetching git dependencies or `-Zgitoxide=fetch,shallow_index` for fetching registry index.
* To enable shallow clones, add `-Zgit=shallow-deps` for fetching git dependencies or `-Zgit=shallow-index` for fetching registry index.
* Shallow-cloned and shallow-checked-out git repositories reside at their own `-shallow` suffixed directories, i.e,
- `~/.cargo/registry/index/*-shallow`
- `~/.cargo/git/db/*-shallow`
- `~/.cargo/git/checkouts/*-shallow`
* When the unstable feature is on, fetching/cloning a git repository is always a shallow fetch. This roughly equals to `git fetch --depth 1` everywhere.
* Even with the presence of `Cargo.lock` or specifying a commit `{ rev = "…" }`, gitoxide is still smart enough to shallow fetch without unshallowing the existing repository.
* Even with the presence of `Cargo.lock` or specifying a commit `{ rev = "…" }`, gitoxide is still smart enough to shallow fetch without unshallowing the existing repository. TODO: does this hold for libgit2?

## script

Expand Down
Loading

0 comments on commit a1559be

Please sign in to comment.