From 984ffec224193781dc6f60df159631adc7af7b15 Mon Sep 17 00:00:00 2001 From: Michael Lieberman Date: Fri, 12 Apr 2024 16:25:20 +0000 Subject: [PATCH] Add update project functionality This will rerun facets against the project and push out any changes. --- skootrs-bin/src/helpers.rs | 32 ++++- skootrs-bin/src/main.rs | 19 +++ skootrs-lib/src/service/facet.rs | 185 ++++++++++++++++++----------- skootrs-lib/src/service/project.rs | 58 ++++++++- skootrs-model/src/skootrs/mod.rs | 11 ++ 5 files changed, 231 insertions(+), 74 deletions(-) diff --git a/skootrs-bin/src/helpers.rs b/skootrs-bin/src/helpers.rs index c0b03b9..078b77e 100644 --- a/skootrs-bin/src/helpers.rs +++ b/skootrs-bin/src/helpers.rs @@ -7,7 +7,7 @@ use skootrs_model::skootrs::{ GithubRepoParams, GithubUser, GoParams, InitializedProject, MavenParams, ProjectArchiveParams, ProjectCreateParams, ProjectGetParams, ProjectOutput, ProjectOutputGetParams, ProjectOutputReference, ProjectOutputType, ProjectOutputsListParams, ProjectReleaseParam, - RepoCreateParams, SkootError, SourceInitializeParams, SupportedEcosystems, + ProjectUpdateParams, RepoCreateParams, SkootError, SourceInitializeParams, SupportedEcosystems, }; use std::{ collections::{HashMap, HashSet}, @@ -169,6 +169,36 @@ impl Project { }) } + /// Updates an existing initialized project to include any updated facets. + /// + /// # Errors + /// + /// Returns an error if the project can't be updated for some reason. + pub async fn update<'a, T: ProjectService + ?Sized>( + config: &Config, + project_service: &'a T, + project_update_params: Option, + ) -> Result { + let mut cache = InMemoryProjectReferenceCache::load_or_create("./skootcache")?; + let project_update_params = match project_update_params { + Some(p) => p, + None => Project::prompt_update(config, project_service).await?, + }; + let updated_project = project_service.update(project_update_params).await?; + cache.set(updated_project.repo.full_url()).await?; + Ok(updated_project) + } + + async fn prompt_update<'a, T: ProjectService + ?Sized>( + config: &Config, + project_service: &'a T, + ) -> Result { + let initialized_project = Project::get(config, project_service, None).await?; + Ok(ProjectUpdateParams { + initialized_project, + }) + } + /// Returns the list of projects that are stored in the cache. /// /// # Errors diff --git a/skootrs-bin/src/main.rs b/skootrs-bin/src/main.rs index d551768..899cc9c 100644 --- a/skootrs-bin/src/main.rs +++ b/skootrs-bin/src/main.rs @@ -109,6 +109,15 @@ enum ProjectCommands { input: Option, }, + /// Update a project. + #[command(name = "update")] + Update { + /// This is an optional input parameter that can be used to pass in a file, pipe, url, or stdin. + /// This is expected to be YAML or JSON. If it is not provided, the CLI will prompt the user for the input. + #[clap(value_parser)] + input: Option, + }, + /// Archive a project. #[command(name = "archive")] Archive { @@ -270,6 +279,16 @@ async fn main() -> std::result::Result<(), SkootError> { error!(error = error.as_ref(), "Failed to get project info"); } } + ProjectCommands::Update { input } => { + let project_update_params = parse_optional_input(input)?; + if let Err(ref error) = + helpers::Project::update(&config, &project_service, project_update_params) + .await + .handle_response_output(stdout()) + { + error!(error = error.as_ref(), "Failed to update project"); + } + } ProjectCommands::List => { if let Err(ref error) = helpers::Project::list(&config) .await diff --git a/skootrs-lib/src/service/facet.rs b/skootrs-lib/src/service/facet.rs index 2dba523..663adad 100644 --- a/skootrs-lib/src/service/facet.rs +++ b/skootrs-lib/src/service/facet.rs @@ -28,17 +28,30 @@ use chrono::Datelike; use tracing::info; +use crate::service::source::SourceService; use skootrs_model::{ security_insights::insights10::{ - SecurityInsightsVersion100YamlSchema, SecurityInsightsVersion100YamlSchemaContributionPolicy, SecurityInsightsVersion100YamlSchemaDependencies, SecurityInsightsVersion100YamlSchemaDependenciesSbomItem, SecurityInsightsVersion100YamlSchemaDependenciesSbomItemSbomCreation, SecurityInsightsVersion100YamlSchemaHeader, SecurityInsightsVersion100YamlSchemaHeaderSchemaVersion, SecurityInsightsVersion100YamlSchemaProjectLifecycle, SecurityInsightsVersion100YamlSchemaProjectLifecycleStatus, SecurityInsightsVersion100YamlSchemaVulnerabilityReporting + SecurityInsightsVersion100YamlSchema, + SecurityInsightsVersion100YamlSchemaContributionPolicy, + SecurityInsightsVersion100YamlSchemaDependencies, + SecurityInsightsVersion100YamlSchemaDependenciesSbomItem, + SecurityInsightsVersion100YamlSchemaDependenciesSbomItemSbomCreation, + SecurityInsightsVersion100YamlSchemaHeader, + SecurityInsightsVersion100YamlSchemaHeaderSchemaVersion, + SecurityInsightsVersion100YamlSchemaProjectLifecycle, + SecurityInsightsVersion100YamlSchemaProjectLifecycleStatus, + SecurityInsightsVersion100YamlSchemaVulnerabilityReporting, }, skootrs::{ facet::{ - APIBundleFacet, APIBundleFacetParams, APIContent, CommonFacetCreateParams, FacetCreateParams, FacetSetCreateParams, InitializedFacet, SourceBundleFacet, SourceBundleFacetCreateParams, SourceFile, SourceFileContent, SourceFileFacet, SourceFileFacetParams, SupportedFacetType - }, InitializedEcosystem, InitializedGithubRepo, InitializedRepo, SkootError + APIBundleFacet, APIBundleFacetParams, APIContent, CommonFacetCreateParams, + FacetCreateParams, FacetSetCreateParams, InitializedFacet, SourceBundleFacet, + SourceBundleFacetCreateParams, SourceFile, SourceFileContent, SourceFileFacet, + SourceFileFacetParams, SupportedFacetType, + }, + InitializedEcosystem, InitializedGithubRepo, InitializedRepo, SkootError, }, }; -use crate::service::source::SourceService; use super::source::LocalSourceService; @@ -50,12 +63,18 @@ pub struct LocalFacetService {} /// This includes things like initializing and managing source files, source bundles, and API bundles. /// It is the root service for all facets and handles which other services to delegate to. pub trait RootFacetService { - fn initialize(&self, params: FacetCreateParams) -> impl std::future::Future> + Send; - fn initialize_all(&self, params: FacetSetCreateParams) -> impl std::future::Future, SkootError>> + Send; + fn initialize( + &self, + params: FacetCreateParams, + ) -> impl std::future::Future> + Send; + fn initialize_all( + &self, + params: FacetSetCreateParams, + ) -> impl std::future::Future, SkootError>> + Send; } -/// (DEPRECATED) The `SourceFileFacetService` trait provides an interface for initializing and managing a project's source -/// file facets. This includes things like initializing and managing READMEs, licenses, and security policy +/// (DEPRECATED) The `SourceFileFacetService` trait provides an interface for initializing and managing a project's source +/// file facets. This includes things like initializing and managing READMEs, licenses, and security policy /// files. /// pub trait SourceFileFacetService { @@ -69,7 +88,7 @@ pub trait SourceFileFacetService { /// The `SourceBundleFacetService` trait provides an interface for initializing and managing a project's source /// bundle facets. This includes things like initializing and managing set of files. -/// +/// /// This replaces the `SourceFileFacetService` trait since it's more generic and can handle more than just /// single files. pub trait SourceBundleFacetService { @@ -124,12 +143,18 @@ impl SourceBundleFacetService for LocalFacetService { } SupportedFacetType::PublishPackages => todo!(), SupportedFacetType::PinnedDependencies => todo!(), - SupportedFacetType::SAST => default_source_bundle_content_handler.generate_content(¶ms)?, + SupportedFacetType::SAST => { + default_source_bundle_content_handler.generate_content(¶ms)? + } SupportedFacetType::VulnerabilityScanner => todo!(), SupportedFacetType::GUACForwardingConfig => todo!(), SupportedFacetType::Allstar => todo!(), - SupportedFacetType::DefaultSourceCode => language_specific_source_bundle_content_handler.generate_content(¶ms)?, - SupportedFacetType::VulnerabilityReporting => unimplemented!("VulnerabilityReporting is not implemented for source bundles"), + SupportedFacetType::DefaultSourceCode => { + language_specific_source_bundle_content_handler.generate_content(¶ms)? + } + SupportedFacetType::VulnerabilityReporting => { + unimplemented!("VulnerabilityReporting is not implemented for source bundles") + } SupportedFacetType::Other => todo!(), }; @@ -146,22 +171,26 @@ impl SourceBundleFacetService for LocalFacetService { )?; } - let source_files: Vec = source_bundle_content.source_files_content.iter().map(|source_file_content| { - Ok::(SourceFile { - name: source_file_content.name.clone(), - path: source_file_content.path.clone(), - hash: source_service.hash_file( - ¶ms.common.source, - source_file_content.path.clone(), - source_file_content.name.clone(), - )?, + let source_files: Vec = source_bundle_content + .source_files_content + .iter() + .map(|source_file_content| { + Ok::(SourceFile { + name: source_file_content.name.clone(), + path: source_file_content.path.clone(), + hash: source_service.hash_file( + ¶ms.common.source, + source_file_content.path.clone(), + source_file_content.name.clone(), + )?, + }) }) - }).collect::, _>>()?; + .collect::, _>>()?; let source_bundle_facet = SourceBundleFacet { source_files: Some(source_files), facet_type: params.facet_type, - source_files_content: None + source_files_content: None, }; Ok(source_bundle_facet) @@ -170,7 +199,7 @@ impl SourceBundleFacetService for LocalFacetService { /// The `APIBundleFacetService` trait provides an interface for initializing and managing a project's API /// bundle facets. This includes things like initializing and managing API calls to services like Github. -/// +/// /// These API calls are used to enable features like branch protection, vulnerability reporting, etc. pub trait APIBundleFacetService { fn initialize( @@ -180,20 +209,17 @@ pub trait APIBundleFacetService { } impl APIBundleFacetService for LocalFacetService { - async fn initialize( - &self, - params: APIBundleFacetParams, - ) -> Result { + async fn initialize(&self, params: APIBundleFacetParams) -> Result { // TODO: This should support more than just Github match params.facet_type { - SupportedFacetType::CodeReview | SupportedFacetType::BranchProtection | SupportedFacetType::VulnerabilityReporting => { + SupportedFacetType::CodeReview + | SupportedFacetType::BranchProtection + | SupportedFacetType::VulnerabilityReporting => { let github_api_bundle_handler = GithubAPIBundleHandler {}; - let api_bundle_facet = - github_api_bundle_handler.generate(¶ms).await?; + let api_bundle_facet = github_api_bundle_handler.generate(¶ms).await?; Ok(api_bundle_facet) } _ => todo!("Not implemented yet"), - } } } @@ -219,7 +245,7 @@ impl RootFacetService for LocalFacetService { FacetCreateParams::APIBundle(params) => { let api_bundle_facet = APIBundleFacetService::initialize(self, params).await?; Ok(InitializedFacet::APIBundle(api_bundle_facet)) - }, + } } } @@ -228,25 +254,20 @@ impl RootFacetService for LocalFacetService { params: FacetSetCreateParams, ) -> Result, SkootError> { let futures = params - .facets_params - .iter() - .map(move |params| RootFacetService::initialize(self, params.clone()) ); + .facets_params + .iter() + .map(move |params| RootFacetService::initialize(self, params.clone())); let results = futures::future::try_join_all(futures).await?; Ok(results) } - - } /// The `APIBundleHandler` trait provides an interface for generating an `APIBundleFacet`. /// This includes calling APIs to services like Github to enable features like branch protection, /// vulnerability reporting, etc. trait APIBundleHandler { - async fn generate( - &self, - params: &APIBundleFacetParams, - ) -> Result; + async fn generate(&self, params: &APIBundleFacetParams) -> Result; } /// The `GithubAPIBundleHandler` struct represents a handler for generating an `APIBundleFacet` related to @@ -254,14 +275,13 @@ trait APIBundleHandler { struct GithubAPIBundleHandler {} impl APIBundleHandler for GithubAPIBundleHandler { - async fn generate( - &self, - params: &APIBundleFacetParams, - ) -> Result { + async fn generate(&self, params: &APIBundleFacetParams) -> Result { let InitializedRepo::Github(repo) = ¶ms.common.repo; match params.facet_type { SupportedFacetType::BranchProtection => self.generate_branch_protection(repo).await, - SupportedFacetType::VulnerabilityReporting => self.generate_vulnerability_reporting(repo).await, + SupportedFacetType::VulnerabilityReporting => { + self.generate_vulnerability_reporting(repo).await + } _ => todo!("Not implemented yet"), } } @@ -278,7 +298,10 @@ impl GithubAPIBundleHandler { repo = repo.name, branch = "main", ); - info!("Enabling branch protection for {}", enforce_branch_protection_endpoint); + info!( + "Enabling branch protection for {}", + enforce_branch_protection_endpoint + ); // TODO: This should be a struct that serializes to json instead of just json directly let enforce_branch_protection_body = serde_json::json!({ "enforce_admins": true, @@ -290,8 +313,18 @@ impl GithubAPIBundleHandler { "allow_deletions": null, }); + // FIXME: I don't quite know why in some cases octocrab loses my auth and I have to re-authenticate + let o: octocrab::Octocrab = octocrab::Octocrab::builder() + .personal_token( + std::env::var("GITHUB_TOKEN").expect("GITHUB_TOKEN env var must be populated"), + ) + .build()?; + octocrab::initialise(o); let response: serde_json::Value = octocrab::instance() - .put(&enforce_branch_protection_endpoint, Some(&enforce_branch_protection_body)) + .put( + &enforce_branch_protection_endpoint, + Some(&enforce_branch_protection_body), + ) .await?; let apis = vec![APIContent { @@ -315,7 +348,10 @@ impl GithubAPIBundleHandler { owner = repo.organization.get_name(), repo = repo.name, ); - info!("Enabling vulnerability reporting for {}", &vulnerability_reporting_endpoint); + info!( + "Enabling vulnerability reporting for {}", + &vulnerability_reporting_endpoint + ); // Note: This call just returns a status with no JSON output also the normal .put I think expects json // output and will fail. octocrab::instance() @@ -326,8 +362,10 @@ impl GithubAPIBundleHandler { url: vulnerability_reporting_endpoint.clone(), response: "Success".to_string(), }]; - info!("Vulnerability reporting enabled for {}", &vulnerability_reporting_endpoint); - + info!( + "Vulnerability reporting enabled for {}", + &vulnerability_reporting_endpoint + ); Ok(APIBundleFacet { facet_type: SupportedFacetType::VulnerabilityReporting, @@ -672,7 +710,7 @@ impl GoGithubSourceBundleContentHandler { project_name: String, module_name: String, } - + #[allow(clippy::match_wildcard_for_single_variants)] let module = match ¶ms.common.ecosystem { InitializedEcosystem::Go(go) => go.module(), @@ -689,21 +727,23 @@ impl GoGithubSourceBundleContentHandler { }; Ok(SourceBundleContent { - source_files_content: vec![SourceFileContent { - name: "releases.yml".to_string(), - path: ".github/workflows/".to_string(), - content: slsa_build_template_params.render()?, - }, - SourceFileContent { - name: "Dockerfile.goreleaser".to_string(), - path: "./".to_string(), - content: dockerfile_template_params.render()?, - }, - SourceFileContent { - name: ".goreleaser.yml".to_string(), - path: "./".to_string(), - content: goreleaser_template_params.render()?, - }], + source_files_content: vec![ + SourceFileContent { + name: "releases.yml".to_string(), + path: ".github/workflows/".to_string(), + content: slsa_build_template_params.render()?, + }, + SourceFileContent { + name: "Dockerfile.goreleaser".to_string(), + path: "./".to_string(), + content: dockerfile_template_params.render()?, + }, + SourceFileContent { + name: ".goreleaser.yml".to_string(), + path: "./".to_string(), + content: goreleaser_template_params.render()?, + }, + ], facet_type: SupportedFacetType::SLSABuild, }) } @@ -797,7 +837,8 @@ impl FacetSetParamsGenerator { &self, common_params: &CommonFacetCreateParams, ) -> Result { - let source_bundle_params = self.generate_default_source_bundle_facet_params(common_params)?; + let source_bundle_params = + self.generate_default_source_bundle_facet_params(common_params)?; let api_bundle_params = self.generate_default_api_bundle(common_params)?; let total_params = FacetSetCreateParams { facets_params: [ @@ -849,8 +890,8 @@ impl FacetSetParamsGenerator { common_params: &CommonFacetCreateParams, ) -> Result { use SupportedFacetType::{ - DefaultSourceCode, DependencyUpdateTool, Gitignore, License, Readme, - SLSABuild, Scorecard, SecurityInsights, SecurityPolicy, SAST, + DefaultSourceCode, DependencyUpdateTool, Gitignore, License, Readme, SLSABuild, + Scorecard, SecurityInsights, SecurityPolicy, SAST, }; let supported_facets = [ Readme, @@ -889,4 +930,4 @@ impl FacetSetParamsGenerator { Ok(FacetSetCreateParams { facets_params }) } -} \ No newline at end of file +} diff --git a/skootrs-lib/src/service/project.rs b/skootrs-lib/src/service/project.rs index 0ef2ec7..e2b6014 100644 --- a/skootrs-lib/src/service/project.rs +++ b/skootrs-lib/src/service/project.rs @@ -23,7 +23,7 @@ use skootrs_model::skootrs::{ facet::{CommonFacetCreateParams, InitializedFacet, SourceFile}, FacetGetParams, FacetMapKey, InitializedProject, InitializedSource, ProjectArchiveParams, ProjectCreateParams, ProjectGetParams, ProjectOutput, ProjectOutputGetParams, - ProjectOutputReference, ProjectOutputsListParams, SkootError, + ProjectOutputReference, ProjectOutputsListParams, ProjectUpdateParams, SkootError, }; use super::{ @@ -88,6 +88,11 @@ pub trait ProjectService { _params: ProjectOutputGetParams, ) -> impl std::future::Future> + Send; + fn update( + &self, + params: ProjectUpdateParams, + ) -> impl std::future::Future> + Send; + /// Archives an initialized project. /// /// # Errors @@ -181,6 +186,7 @@ where ecosystem: initialized_ecosystem, source: initialized_source, facets: initialized_facets, + name: params.name.clone(), }) } @@ -256,6 +262,56 @@ where } } + // TODO: A lot of this code is copied from the initialize function. This should be refactored to avoid code duplication. + async fn update(&self, params: ProjectUpdateParams) -> Result { + let initialized_project = params.initialized_project.clone(); + let initialized_repo = initialized_project.repo; + let initialized_source = self.repo_service.clone_local_or_pull( + initialized_repo.clone(), + initialized_project.source.path.clone(), + )?; + let initialized_ecosystem = initialized_project.ecosystem; + + let facet_set_params_generator = FacetSetParamsGenerator {}; + let common_params = CommonFacetCreateParams { + project_name: initialized_project.name.clone(), + source: initialized_source.clone(), + repo: initialized_repo.clone(), + ecosystem: initialized_ecosystem.clone(), + }; + let source_facet_set_params = facet_set_params_generator + .generate_default_source_bundle_facet_params(&common_params)?; + let api_facet_set_params = + facet_set_params_generator.generate_default_api_bundle(&common_params)?; + let initialized_source_facets = self + .facet_service + .initialize_all(source_facet_set_params) + .await?; + // TODO: Figure out how to better order commits and pushes + self.source_service.commit_and_push_changes( + initialized_source.clone(), + "Updated facets for project".to_string(), + )?; + let initialized_api_facets = self + .facet_service + .initialize_all(api_facet_set_params) + .await?; + // FIXME: Also add facet by name as well + let initialized_facets = [initialized_source_facets, initialized_api_facets] + .concat() + .into_iter() + .map(|f| (FacetMapKey::Type(f.facet_type()), f)) + .collect::>(); + + Ok(InitializedProject { + repo: initialized_repo, + ecosystem: initialized_ecosystem, + source: initialized_source, + facets: initialized_facets, + name: initialized_project.name.clone(), + }) + } + async fn outputs_list( &self, params: ProjectOutputsListParams, diff --git a/skootrs-model/src/skootrs/mod.rs b/skootrs-model/src/skootrs/mod.rs index 64e8d75..6748692 100644 --- a/skootrs-model/src/skootrs/mod.rs +++ b/skootrs-model/src/skootrs/mod.rs @@ -65,6 +65,9 @@ pub struct InitializedProject { pub source: InitializedSource, /// The facets associated with the project. pub facets: HashMap, + // TODO: What to do if there are name collisions? + /// The name of the project. + pub name: String, } /// A helper enum for how a facet can be pulled from a `HashMap` @@ -142,6 +145,14 @@ pub struct ProjectCreateParams { pub source_params: SourceInitializeParams, } +/// The parameters for updating a project. +#[derive(Serialize, Deserialize, Clone, Debug)] +#[cfg_attr(feature = "openapi", derive(ToSchema))] +pub struct ProjectUpdateParams { + /// The initialized project to update. + pub initialized_project: InitializedProject, +} + /// The parameters for getting an existing Skootrs project. #[derive(Serialize, Deserialize, Clone, Debug)] #[cfg_attr(feature = "openapi", derive(ToSchema))]