Skip to content

Commit

Permalink
Auto merge of #13252 - connorworley:shallow_libgit2_fetches, r=weihanglo
Browse files Browse the repository at this point in the history
Implementation of shallow libgit2 fetches behind an unstable flag

This is my first contribution, so guidance is appreciated.

Fixes #1171 by moving the `shallow-index` and `shallow-deps` aspects of `-Zgitoxide` to a new `-Zgit` unstable flag.

The only change in interaction with libgit2 happens in `src/cargo/sources/git/utils.rs`, where we set the depth fetch option if applicable.

Shallow fetch tests for gitoxide continue to pass, but libgit2 is harder to test as it silently ignores the depth option for local fetches. I would love any ideas on how to test it in a lightweight way or whether it's OK to wait for an upstream fix.
  • Loading branch information
bors committed Jan 12, 2024
2 parents 54fd66a + 426b9e3 commit bf27f90
Show file tree
Hide file tree
Showing 11 changed files with 155 additions and 60 deletions.
57 changes: 45 additions & 12 deletions src/cargo/core/features.rs
Original file line number Diff line number Diff line change
Expand Up @@ -770,6 +770,7 @@ unstable_cli_options!(
dual_proc_macros: bool = ("Build proc-macros for both the host and the target"),
features: Option<Vec<String>>,
gc: bool = ("Track cache usage and \"garbage collect\" unused files"),
git: Option<GitFeatures> = ("Enable support for shallow git fetch operations"),
gitoxide: Option<GitoxideFeatures> = ("Use gitoxide for the given git interactions, or all of them if no argument is given"),
host_config: bool = ("Enable the `[host]` section in the .cargo/config.toml file"),
lints: bool = ("Pass `[lints]` to the linting tools"),
Expand Down Expand Up @@ -875,13 +876,47 @@ where
}

#[derive(Debug, Copy, Clone, Default, Deserialize)]
pub struct GitoxideFeatures {
/// All fetches are done with `gitoxide`, which includes git dependencies as well as the crates index.
pub fetch: bool,
pub struct GitFeatures {
/// When cloning the index, perform a shallow clone. Maintain shallowness upon subsequent fetches.
pub shallow_index: bool,
/// When cloning git dependencies, perform a shallow clone and maintain shallowness on subsequent fetches.
pub shallow_deps: bool,
}

impl GitFeatures {
fn all() -> Self {
GitFeatures {
shallow_index: true,
shallow_deps: true,
}
}
}

fn parse_git(it: impl Iterator<Item = impl AsRef<str>>) -> CargoResult<Option<GitFeatures>> {
let mut out = GitFeatures::default();
let GitFeatures {
shallow_index,
shallow_deps,
} = &mut out;

for e in it {
match e.as_ref() {
"shallow-index" => *shallow_index = true,
"shallow-deps" => *shallow_deps = true,
_ => {
bail!(
"unstable 'git' only takes 'shallow-index' and 'shallow-deps' as valid inputs"
)
}
}
}
Ok(Some(out))
}

#[derive(Debug, Copy, Clone, Default, Deserialize)]
pub struct GitoxideFeatures {
/// All fetches are done with `gitoxide`, which includes git dependencies as well as the crates index.
pub fetch: bool,
/// Checkout git dependencies using `gitoxide` (submodules are still handled by git2 ATM, and filters
/// like linefeed conversions are unsupported).
pub checkout: bool,
Expand All @@ -895,9 +930,7 @@ impl GitoxideFeatures {
fn all() -> Self {
GitoxideFeatures {
fetch: true,
shallow_index: true,
checkout: true,
shallow_deps: true,
internal_use_git2: false,
}
}
Expand All @@ -907,9 +940,7 @@ impl GitoxideFeatures {
fn safe() -> Self {
GitoxideFeatures {
fetch: true,
shallow_index: false,
checkout: true,
shallow_deps: false,
internal_use_git2: false,
}
}
Expand All @@ -921,21 +952,17 @@ fn parse_gitoxide(
let mut out = GitoxideFeatures::default();
let GitoxideFeatures {
fetch,
shallow_index,
checkout,
shallow_deps,
internal_use_git2,
} = &mut out;

for e in it {
match e.as_ref() {
"fetch" => *fetch = true,
"shallow-index" => *shallow_index = true,
"shallow-deps" => *shallow_deps = true,
"checkout" => *checkout = true,
"internal-use-git2" => *internal_use_git2 = true,
_ => {
bail!("unstable 'gitoxide' only takes `fetch`, 'shallow-index', 'shallow-deps' and 'checkout' as valid inputs")
bail!("unstable 'gitoxide' only takes `fetch` and 'checkout' as valid input, for shallow fetches see `-Zgit=shallow-index,shallow-deps`")
}
}
}
Expand Down Expand Up @@ -1109,6 +1136,12 @@ impl CliUnstable {
"doctest-xcompile" => self.doctest_xcompile = parse_empty(k, v)?,
"dual-proc-macros" => self.dual_proc_macros = parse_empty(k, v)?,
"gc" => self.gc = parse_empty(k, v)?,
"git" => {
self.git = v.map_or_else(
|| Ok(Some(GitFeatures::all())),
|v| parse_git(v.split(',')),
)?
}
"gitoxide" => {
self.gitoxide = v.map_or_else(
|| Ok(Some(GitoxideFeatures::all())),
Expand Down
11 changes: 6 additions & 5 deletions src/cargo/sources/git/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ mod utils;

/// For `-Zgitoxide` integration.
pub mod fetch {
use crate::core::features::GitoxideFeatures;
use crate::core::features::GitFeatures;
use crate::Config;

/// The kind remote repository to fetch.
Expand All @@ -37,18 +37,19 @@ pub mod fetch {
repo_is_shallow: bool,
config: &Config,
) -> gix::remote::fetch::Shallow {
let has_feature = |cb: &dyn Fn(GitoxideFeatures) -> bool| {
let has_feature = |cb: &dyn Fn(GitFeatures) -> bool| {
config
.cli_unstable()
.gitoxide
.git
.map_or(false, |features| cb(features))
};

// maintain shallow-ness and keep downloading single commits, or see if we can do shallow clones
if !repo_is_shallow {
match self {
RemoteKind::GitDependency if has_feature(&|git| git.shallow_deps) => {}
RemoteKind::Registry if has_feature(&|git| git.shallow_index) => {}
RemoteKind::GitDependency if has_feature(&|features| features.shallow_deps) => {
}
RemoteKind::Registry if has_feature(&|features| features.shallow_index) => {}
_ => return gix::remote::fetch::Shallow::NoChange,
}
};
Expand Down
7 changes: 3 additions & 4 deletions src/cargo/sources/git/oxide.rs
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,9 @@ pub fn with_retry_and_progress(
) -> CargoResult<()> {
std::thread::scope(|s| {
let mut progress_bar = Progress::new("Fetch", config);
let is_shallow = config
.cli_unstable()
.gitoxide
.map_or(false, |gix| gix.shallow_deps || gix.shallow_index);
let is_shallow = config.cli_unstable().git.map_or(false, |features| {
features.shallow_deps || features.shallow_index
});
network::retry::with_retry(config, || {
let progress_root: Arc<gix::progress::tree::Root> =
gix::progress::tree::root::Options {
Expand Down
4 changes: 2 additions & 2 deletions src/cargo/sources/git/source.rs
Original file line number Diff line number Diff line change
Expand Up @@ -108,8 +108,8 @@ impl<'cfg> GitSource<'cfg> {
&source_id,
config
.cli_unstable()
.gitoxide
.map_or(false, |gix| gix.fetch && gix.shallow_deps),
.git
.map_or(false, |features| features.shallow_deps),
);

let source = GitSource {
Expand Down
3 changes: 3 additions & 0 deletions src/cargo/sources/git/utils.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1117,6 +1117,9 @@ pub fn fetch(
if tags {
opts.download_tags(git2::AutotagOption::All);
}
if let gix::remote::fetch::Shallow::DepthAtRemote(depth) = shallow {
opts.depth(0i32.saturating_add_unsigned(depth.get()));
}
// The `fetch` operation here may fail spuriously due to a corrupt
// repository. It could also fail, however, for a whole slew of other
// reasons (aka network related reasons). We want Cargo to automatically
Expand Down
4 changes: 2 additions & 2 deletions src/cargo/sources/registry/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -476,8 +476,8 @@ impl<'cfg> RegistrySource<'cfg> {
source_id,
config
.cli_unstable()
.gitoxide
.map_or(false, |gix| gix.fetch && gix.shallow_index)
.git
.map_or(false, |features| features.shallow_index)
&& !source_id.is_sparse(),
);
let ops = if source_id.is_sparse() {
Expand Down
19 changes: 16 additions & 3 deletions src/doc/src/reference/unstable.md
Original file line number Diff line number Diff line change
Expand Up @@ -1172,19 +1172,32 @@ to run with `gitoxide` with the `-Zgitoxide=operation[,operationN]` syntax.
Valid operations are the following:

* `fetch` - All fetches are done with `gitoxide`, which includes git dependencies as well as the crates index.
* `checkout` *(planned)* - checkout the worktree, with support for filters and submodules.

## git

* Tracking Issue: [#13285](https://github.com/rust-lang/cargo/issues/13285)

With the 'git' unstable feature, both `gitoxide` and `git2` will perform shallow fetches of the crate
index and git dependencies.

While `-Zgit` enables all currently implemented features, one can individually select when to perform
shallow fetches with the `-Zgit=operation[,operationN]` syntax.

Valid operations are the following:

* `shallow-index` - perform a shallow clone of the index.
* `shallow-deps` - perform a shallow clone of git dependencies.
* `checkout` *(planned)* - checkout the worktree, with support for filters and submodules.

**Details on shallow clones**

* To enable shallow clones, add `-Zgitoxide=fetch,shallow_deps` for fetching git dependencies or `-Zgitoxide=fetch,shallow_index` for fetching registry index.
* To enable shallow clones, add `-Zgit=shallow-deps` for fetching git dependencies or `-Zgit=shallow-index` for fetching registry index.
* Shallow-cloned and shallow-checked-out git repositories reside at their own `-shallow` suffixed directories, i.e,
- `~/.cargo/registry/index/*-shallow`
- `~/.cargo/git/db/*-shallow`
- `~/.cargo/git/checkouts/*-shallow`
* When the unstable feature is on, fetching/cloning a git repository is always a shallow fetch. This roughly equals to `git fetch --depth 1` everywhere.
* Even with the presence of `Cargo.lock` or specifying a commit `{ rev = "…" }`, gitoxide is still smart enough to shallow fetch without unshallowing the existing repository.
* Even with the presence of `Cargo.lock` or specifying a commit `{ rev = "…" }`, gitoxide and libgit2 are still smart enough to shallow fetch without unshallowing the existing repository.

## script

Expand Down
1 change: 1 addition & 0 deletions tests/testsuite/cargo/z_help/stdout.log
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ Available unstable (nightly-only) flags:
-Z doctest-xcompile Compile and run doctests for non-host target using runner config
-Z dual-proc-macros Build proc-macros for both the host and the target
-Z gc Track cache usage and "garbage collect" unused files
-Z git Enable support for shallow git fetch operations
-Z gitoxide Use gitoxide for the given git interactions, or all of them if no argument is given
-Z host-config Enable the `[host]` section in the .cargo/config.toml file
-Z lints Pass `[lints]` to the linting tools
Expand Down
Loading

0 comments on commit bf27f90

Please sign in to comment.