From 1abcdcce4c16c5391b4ef13cd420dd8a37d2315b Mon Sep 17 00:00:00 2001 From: leoporoli Date: Fri, 15 Mar 2024 15:57:54 -0300 Subject: [PATCH] fix: the package replace logic when using tag, branch or commit (#2299) ## Description Fix the package replace logic when using tag, branch or commit The current issue is the `package replace logic` fails when you add a tag, branch or commit in the replace value, like this example: `replace: - github.com/kurtosis-tech/ethereum-package: github.com/kurtosis-tech/ethereum-package@da55be84861e93ce777076e545abee35ff2d51ce` If we see the evaluation error `Evaluation error: An error occurred while loading the module 'github.com/kurtosis-tech/ethereum-package@da55be84861e93ce777076e545abee35ff2d51ce/src/package_io/input_parser.star' Caused by: '/input_parser.star' doesn't exist in the package 'kurtosis-tech/ethereum-package' at [github.com/kurtosis-tech/ethereum-package@da55be84861e93ce777076e545abee35ff2d51ce/main.star:1:29]: at [github.com/kurtosis-tech/awesome-kurtosis/chainlink-node/main.star:2:33]: ` we understand it's trying to import the file from this path `github.com/kurtosis-tech/ethereum-package@da55be84861e93ce777076e545abee35ff2d51ce` and it doesn't exist because the folder, inside the APIC file system, is created without the hash, it's `github.com/kurtosis-tech/ethereum-package` This is happening because an absolute locator like this `github.com/kurtosis-tech/ethereum-package/src/package_io/input_parser.star` does not incorporate the concept of hash, branch or commit. The fix is incorporated the concept of `hash, branch or commit` in a new AbsoluteLocator object, so this way we can specify if the absolute locator is pointing to other place than the main branch. With this change we can accept the `github.com/kurtosis-tech/ethereum-package@da55be84861e93ce777076e545abee35ff2d51ce` replace value and an absolute locator string like this `github.com/kurtosis-tech/ethereum-package@da55be84861e93ce777076e545abee35ff2d51ce/src/package_io/input_parser.star` will be represented with an AbsoluteLocator object with locator `github.com/kurtosis-tech/ethereum-package/src/package_io/input_parser.star` and and tagBranchOrCommit `da55be84861e93ce777076e545abee35ff2d51ce` We already had support for retrieving packages from different tag, branch or commit but only for `packageId` values and not for `absolute locators` ## REMINDER: Tag Reviewers, so they get notified to review ## Is this change user facing? YES ## References (if applicable) Fix #2279 --- .../core/lib/shared_utils/parsed_git_url.go | 6 +- .../builtins/import_module/import_module.go | 7 +- .../builtins/read_file/read_file.go | 4 +- .../image_build_spec_framework_test.go | 4 +- ...ild_spec_framework_with_build_args_test.go | 4 +- ...ld_spec_framework_with_build_file_test.go} | 5 +- ...d_spec_framework_with_target_stage_test.go | 4 +- .../import_module_framework_test.go | 4 +- ...h_local_absolute_locator_framework_test.go | 2 +- .../nix_build_spec_framework_test.go | 6 +- .../test_engine/read_file_framework_test.go | 6 +- ...h_local_absolute_locator_framework_test.go | 2 +- .../service_config_image_build_spec_test.go | 4 +- .../test_engine/static_constants.go | 46 +++-- .../service_config/image_build_spec.go | 6 +- .../service_config/nix_build_spec.go | 4 +- .../startosis_packages/absolute_locator.go | 35 ++++ .../git_package_content_provider.go | 35 ++-- .../git_package_content_provider_test.go | 172 +++++++++++++++--- .../git_package_content_provider/locators.go | 26 +-- .../mock_package_content_provider.go | 117 ++++++------ .../mock_package_content_provider.go | 39 ++-- .../package_content_provider.go | 8 +- 23 files changed, 370 insertions(+), 176 deletions(-) rename core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/{image_build_spec_framework_with_build_file.go => image_build_spec_framework_with_build_file_test.go} (92%) create mode 100644 core/server/api_container/server/startosis_engine/startosis_packages/absolute_locator.go diff --git a/api/golang/core/lib/shared_utils/parsed_git_url.go b/api/golang/core/lib/shared_utils/parsed_git_url.go index 215398b652..5f7db032ee 100644 --- a/api/golang/core/lib/shared_utils/parsed_git_url.go +++ b/api/golang/core/lib/shared_utils/parsed_git_url.go @@ -108,7 +108,7 @@ func ParseGitURL(packageURL string) (*ParsedGitURL, error) { return nil, stacktrace.NewError("Error parsing the URL of module. We only support modules on Github for now but got '%v'", packageURL) } - pathWithoutVersion, maybeTagBranchOrCommit := parseOutTagBranchOrCommit(parsedURL.Path) + pathWithoutVersion, maybeTagBranchOrCommit := ParseOutTagBranchOrCommit(parsedURL.Path) splitURLPath := cleanPathAndSplit(pathWithoutVersion) @@ -151,8 +151,8 @@ func cleanPathAndSplit(urlPath string) []string { return sliceWithoutEmptyStrings } -// parseOutTagBranchOrCommit splits the string around "@" and then split the after string around "/" -func parseOutTagBranchOrCommit(input string) (string, string) { +// ParseOutTagBranchOrCommit splits the string around "@" and then split the after string around "/" +func ParseOutTagBranchOrCommit(input string) (string, string) { cleanInput := path.Clean(input) pathWithoutVersion, maybeTagBranchOrCommitWithFile, _ := strings.Cut(cleanInput, tagBranchOrCommitDelimiter) diff --git a/core/server/api_container/server/startosis_engine/builtins/import_module/import_module.go b/core/server/api_container/server/startosis_engine/builtins/import_module/import_module.go index 36d36afa20..7e2351748f 100644 --- a/core/server/api_container/server/startosis_engine/builtins/import_module/import_module.go +++ b/core/server/api_container/server/startosis_engine/builtins/import_module/import_module.go @@ -78,11 +78,12 @@ func (builtin *importModuleCapabilities) Interpret(locatorOfModuleInWhichThisBui return nil, explicitInterpretationError(err) } relativeOrAbsoluteModuleLocator := moduleLocatorArgValue.GoString() - absoluteModuleLocator, relativePathParsingInterpretationErr := builtin.packageContentProvider.GetAbsoluteLocator(builtin.packageId, locatorOfModuleInWhichThisBuiltInIsBeingCalled, relativeOrAbsoluteModuleLocator, builtin.packageReplaceOptions) + absoluteModuleLocatorObj, relativePathParsingInterpretationErr := builtin.packageContentProvider.GetAbsoluteLocator(builtin.packageId, locatorOfModuleInWhichThisBuiltInIsBeingCalled, relativeOrAbsoluteModuleLocator, builtin.packageReplaceOptions) if relativePathParsingInterpretationErr != nil { return nil, relativePathParsingInterpretationErr } - logrus.Debugf("importing module from absolute locator '%s'", absoluteModuleLocator) + absoluteModuleLocator := absoluteModuleLocatorObj.GetLocator() + logrus.Debugf("importing module from absolute locator '%s' with tag, branch or commit %s", absoluteModuleLocator, absoluteModuleLocatorObj.GetTagBranchOrCommit()) var loadInProgress *startosis_packages.ModuleCacheEntry cacheEntry, found := builtin.moduleGlobalCache[absoluteModuleLocator] @@ -102,7 +103,7 @@ func (builtin *importModuleCapabilities) Interpret(locatorOfModuleInWhichThisBui }() // Load it. - contents, interpretationError := builtin.packageContentProvider.GetModuleContents(absoluteModuleLocator) + contents, interpretationError := builtin.packageContentProvider.GetModuleContents(absoluteModuleLocatorObj) if interpretationError != nil { return nil, startosis_errors.WrapWithInterpretationError(interpretationError, "An error occurred while loading the module '%v'", absoluteModuleLocator) } diff --git a/core/server/api_container/server/startosis_engine/builtins/read_file/read_file.go b/core/server/api_container/server/startosis_engine/builtins/read_file/read_file.go index f08d70e3ea..3a23c0462a 100644 --- a/core/server/api_container/server/startosis_engine/builtins/read_file/read_file.go +++ b/core/server/api_container/server/startosis_engine/builtins/read_file/read_file.go @@ -55,11 +55,11 @@ func (builtin *readFileCapabilities) Interpret(locatorOfModuleInWhichThisBuiltIn return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for arg '%s'", srcValue) } fileToReadStr := srcValue.GoString() - fileToReadStr, relativePathParsingInterpretationErr := builtin.packageContentProvider.GetAbsoluteLocator(builtin.packageId, locatorOfModuleInWhichThisBuiltInIsBeingCalled, fileToReadStr, builtin.packageReplaceOptions) + absoluteLocator, relativePathParsingInterpretationErr := builtin.packageContentProvider.GetAbsoluteLocator(builtin.packageId, locatorOfModuleInWhichThisBuiltInIsBeingCalled, fileToReadStr, builtin.packageReplaceOptions) if relativePathParsingInterpretationErr != nil { return nil, relativePathParsingInterpretationErr } - packageContent, interpretationErr := builtin.packageContentProvider.GetModuleContents(fileToReadStr) + packageContent, interpretationErr := builtin.packageContentProvider.GetModuleContents(absoluteLocator) if interpretationErr != nil { return nil, interpretationErr } diff --git a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/image_build_spec_framework_test.go b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/image_build_spec_framework_test.go index a28a4833ca..ee063c7aa9 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/image_build_spec_framework_test.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/image_build_spec_framework_test.go @@ -20,10 +20,10 @@ func (suite *KurtosisTypeConstructorTestSuite) TestImageBuildSpecTest() { suite.packageContentProvider.EXPECT(). GetAbsoluteLocator(testModulePackageId, testModuleMainFileLocator, testBuildContextDir, testNoPackageReplaceOptions). Times(1). - Return(testBuildContextLocator, nil) + Return(testBuildContextAbsoluteLocator, nil) suite.packageContentProvider.EXPECT(). - GetOnDiskAbsolutePackageFilePath(testContainerImageLocator). + GetOnDiskAbsolutePackageFilePath(testContainerImageAbsoluteLocator). Times(1). Return(testOnDiskContainerImagePath, nil) diff --git a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/image_build_spec_framework_with_build_args_test.go b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/image_build_spec_framework_with_build_args_test.go index 3adaa5092b..c8ae2fc030 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/image_build_spec_framework_with_build_args_test.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/image_build_spec_framework_with_build_args_test.go @@ -20,10 +20,10 @@ func (suite *KurtosisTypeConstructorTestSuite) TestImageBuildSpecWithBuildArgsTe suite.packageContentProvider.EXPECT(). GetAbsoluteLocator(testModulePackageId, testModuleMainFileLocator, testBuildContextDir, testNoPackageReplaceOptions). Times(1). - Return(testBuildContextLocator, nil) + Return(testBuildContextAbsoluteLocator, nil) suite.packageContentProvider.EXPECT(). - GetOnDiskAbsolutePackageFilePath(testContainerImageLocator). + GetOnDiskAbsolutePackageFilePath(testContainerImageAbsoluteLocator). Times(1). Return(testOnDiskContainerImagePath, nil) diff --git a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/image_build_spec_framework_with_build_file.go b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/image_build_spec_framework_with_build_file_test.go similarity index 92% rename from core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/image_build_spec_framework_with_build_file.go rename to core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/image_build_spec_framework_with_build_file_test.go index c07bd5d40c..042d81d601 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/image_build_spec_framework_with_build_file.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/image_build_spec_framework_with_build_file_test.go @@ -20,10 +20,10 @@ func (suite *KurtosisTypeConstructorTestSuite) TestImageBuildSpecTestWithBuildFi suite.packageContentProvider.EXPECT(). GetAbsoluteLocator(testModulePackageId, testModuleMainFileLocator, testBuildContextDir, testNoPackageReplaceOptions). Times(1). - Return(testBuildContextLocator, nil) + Return(testModulePackageAbsoluteLocator, nil) suite.packageContentProvider.EXPECT(). - GetOnDiskAbsolutePackageFilePath(testContainerImageLocatorWithBuildFile). + GetOnDiskAbsolutePackageFilePath(testContainerImageAbsoluteLocatorWithBuildFile). Times(1). Return(testOnDiskContainerImagePathWithBuildFile, nil) @@ -56,5 +56,4 @@ func (t *imageBuildSpecWithBuildFileTest) Assert(typeValue builtin_argument.Kurt require.Nil(t, err) require.Equal(t, testOnDiskContainerImagePathWithBuildFile, imageBuildSpec.GetContainerImageFilePath()) require.Equal(t, testOnDiskContextDirPath, imageBuildSpec.GetBuildContextDir()) - //require.Equal(t, "", imageBuildSpec.GetTargetStage()) } diff --git a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/image_build_spec_framework_with_target_stage_test.go b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/image_build_spec_framework_with_target_stage_test.go index bd46adf480..5bd7beb892 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/image_build_spec_framework_with_target_stage_test.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/image_build_spec_framework_with_target_stage_test.go @@ -20,10 +20,10 @@ func (suite *KurtosisTypeConstructorTestSuite) TestImageBuildSpecWithTargetStage suite.packageContentProvider.EXPECT(). GetAbsoluteLocator(testModulePackageId, testModuleMainFileLocator, testBuildContextDir, testNoPackageReplaceOptions). Times(1). - Return(testBuildContextLocator, nil) + Return(testModulePackageAbsoluteLocator, nil) suite.packageContentProvider.EXPECT(). - GetOnDiskAbsolutePackageFilePath(testContainerImageLocator). + GetOnDiskAbsolutePackageFilePath(testContainerImageAbsoluteLocator). Times(1). Return(testOnDiskContainerImagePath, nil) diff --git a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/import_module_framework_test.go b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/import_module_framework_test.go index 801a418ff7..3ae1fff47d 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/import_module_framework_test.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/import_module_framework_test.go @@ -38,8 +38,8 @@ func (suite *KurtosisHelperTestSuite) TestImportFile() { // start with an empty cache to validate it gets populated moduleGlobalCache := map[string]*startosis_packages.ModuleCacheEntry{} - suite.packageContentProvider.EXPECT().GetModuleContents(testModuleFileName).Return("Hello World!", nil) - suite.packageContentProvider.EXPECT().GetAbsoluteLocator(testModulePackageId, startosis_constants.PackageIdPlaceholderForStandaloneScript, testModuleRelativeLocator, testNoPackageReplaceOptions).Return(testModuleFileName, nil) + suite.packageContentProvider.EXPECT().GetModuleContents(testModuleAbsoluteLocator).Return("Hello World!", nil) + suite.packageContentProvider.EXPECT().GetAbsoluteLocator(testModulePackageId, startosis_constants.PackageIdPlaceholderForStandaloneScript, testModuleRelativeLocator, testNoPackageReplaceOptions).Return(testModuleAbsoluteLocator, nil) suite.run(&importModuleTestCase{ T: suite.T(), diff --git a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/import_module_with_local_absolute_locator_framework_test.go b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/import_module_with_local_absolute_locator_framework_test.go index b9cd9c149f..d56f3974da 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/import_module_with_local_absolute_locator_framework_test.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/import_module_with_local_absolute_locator_framework_test.go @@ -33,7 +33,7 @@ type importModuleWithLocalAbsoluteLocatorTestCase struct { } func (suite *KurtosisHelperTestSuite) TestImportFileWithLocalAbsoluteLocatorShouldNotBeValid() { - suite.packageContentProvider.EXPECT().GetAbsoluteLocator(testModulePackageId, testModuleMainFileLocator, testModuleFileName, testNoPackageReplaceOptions).Return("", startosis_errors.NewInterpretationError(importModuleWithLocalAbsoluteLocatorExpectedErrorMsg)) + suite.packageContentProvider.EXPECT().GetAbsoluteLocator(testModulePackageId, testModuleMainFileLocator, testModuleFileName, testNoPackageReplaceOptions).Return(emptyAbsoluteLocator, startosis_errors.NewInterpretationError(importModuleWithLocalAbsoluteLocatorExpectedErrorMsg)) // start with an empty cache to validate it gets populated moduleGlobalCache := map[string]*startosis_packages.ModuleCacheEntry{} diff --git a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/nix_build_spec_framework_test.go b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/nix_build_spec_framework_test.go index f4c5909dd2..b15c184811 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/nix_build_spec_framework_test.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/nix_build_spec_framework_test.go @@ -20,15 +20,15 @@ func (suite *KurtosisTypeConstructorTestSuite) TestNixBuildSpecTest() { suite.packageContentProvider.EXPECT(). GetAbsoluteLocator(testModulePackageId, testModuleMainFileLocator, testBuildContextDir, testNoPackageReplaceOptions). Times(1). - Return(testBuildContextLocator, nil) + Return(testModulePackageAbsoluteLocator, nil) suite.packageContentProvider.EXPECT(). - GetOnDiskAbsolutePackageFilePath(testNixFlakeLocator). + GetOnDiskAbsolutePackageFilePath(testNixFlakeAbsoluteLocator). Times(1). Return(testOnDiskNixFlakePath, nil) suite.packageContentProvider.EXPECT(). - GetOnDiskAbsolutePath(testBuildContextLocator). + GetOnDiskAbsolutePath(testBuildContextAbsoluteLocator). Times(1). Return(testOnDiskContextDirPath, nil) diff --git a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/read_file_framework_test.go b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/read_file_framework_test.go index d3b6c4652a..b9f1b383f0 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/read_file_framework_test.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/read_file_framework_test.go @@ -18,8 +18,8 @@ type readFileTestCase struct { } func (suite *KurtosisHelperTestSuite) TestReadFile() { - suite.packageContentProvider.EXPECT().GetAbsoluteLocator(testModulePackageId, startosis_constants.PackageIdPlaceholderForStandaloneScript, testModuleRelativeLocator, testNoPackageReplaceOptions).Return(testModuleFileName, nil) - suite.packageContentProvider.EXPECT().GetModuleContents(testModuleFileName).Return("Hello World!", nil) + suite.packageContentProvider.EXPECT().GetAbsoluteLocator(testModulePackageId, startosis_constants.PackageIdPlaceholderForStandaloneScript, testModuleRelativeLocator, testNoPackageReplaceOptions).Return(testModuleAbsoluteLocator, nil) + suite.packageContentProvider.EXPECT().GetModuleContents(testModuleAbsoluteLocator).Return("Hello World!", nil) suite.run(&readFileTestCase{ T: suite.T(), @@ -41,6 +41,6 @@ func (t *readFileTestCase) GetStarlarkCodeForAssertion() string { func (t *readFileTestCase) Assert(result starlark.Value) { t.packageContentProvider.AssertCalled(t, "GetAbsoluteLocator", testModulePackageId, startosis_constants.PackageIdPlaceholderForStandaloneScript, testModuleRelativeLocator, testNoPackageReplaceOptions) - t.packageContentProvider.AssertCalled(t, "GetModuleContents", testModuleFileName) + t.packageContentProvider.AssertCalled(t, "GetModuleContents", testModuleAbsoluteLocator) require.Equal(t, result, starlark.String("Hello World!")) } diff --git a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/read_file_with_local_absolute_locator_framework_test.go b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/read_file_with_local_absolute_locator_framework_test.go index 7a2d08ab93..3ded47343d 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/read_file_with_local_absolute_locator_framework_test.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/read_file_with_local_absolute_locator_framework_test.go @@ -21,7 +21,7 @@ type readFileWithLocalAbsoluteLocatorTestCase struct { } func (suite *KurtosisHelperTestSuite) TestReadFileWithLocalAbsoluteLocatorShouldNotBeValid() { - suite.packageContentProvider.EXPECT().GetAbsoluteLocator(testModulePackageId, testModuleMainFileLocator, testModuleFileName, testNoPackageReplaceOptions).Return("", startosis_errors.NewInterpretationError(readFileWithLocalAbsoluteLocatorExpectedErrorMsg)) + suite.packageContentProvider.EXPECT().GetAbsoluteLocator(testModulePackageId, testModuleMainFileLocator, testModuleFileName, testNoPackageReplaceOptions).Return(emptyAbsoluteLocator, startosis_errors.NewInterpretationError(readFileWithLocalAbsoluteLocatorExpectedErrorMsg)) suite.runShouldFail( testModuleMainFileLocator, diff --git a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/service_config_image_build_spec_test.go b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/service_config_image_build_spec_test.go index d5f3a13bfc..fbecb71f13 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/service_config_image_build_spec_test.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/service_config_image_build_spec_test.go @@ -25,10 +25,10 @@ func (suite *KurtosisTypeConstructorTestSuite) TestServiceConfigWithImageBuildSp suite.packageContentProvider.EXPECT(). GetAbsoluteLocator(testModulePackageId, testModuleMainFileLocator, testBuildContextDir, testNoPackageReplaceOptions). Times(1). - Return(testBuildContextLocator, nil) + Return(testModulePackageAbsoluteLocator, nil) suite.packageContentProvider.EXPECT(). - GetOnDiskAbsolutePackageFilePath(testContainerImageLocator). + GetOnDiskAbsolutePackageFilePath(testContainerImageAbsoluteLocator). Times(1). Return(testOnDiskContainerImagePath, nil) diff --git a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/static_constants.go b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/static_constants.go index f71928c266..c94026fc3f 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/static_constants.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/static_constants.go @@ -4,6 +4,7 @@ import ( "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/enclave" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/port_spec" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_packages" "github.com/kurtosis-tech/kurtosis/core/server/commons/enclave_data_directory" ) @@ -21,25 +22,30 @@ var ( testSrcPath = "/path/to/file.txt" - testModulePackageId = "github.com/kurtosistech/test-package" - testModuleMainFileLocator = "github.com/kurtosistech/test-package/main.star" - testModuleFileName = "github.com/kurtosistech/test-package/helpers.star" - testModuleRelativeLocator = "./helpers.star" - - testContainerImageName = "kurtosistech/example-datastore-server" - testBuildContextDir = "./" - testBuildFile = "foo.Dockerfile" - testTargetStage = "builder" - testBuildArgName1 = "BUILD_ARG_1" - testBuildArgValue1 = "VALUE_1" - testBuildArgName2 = "BUILD_ARG_2" - testBuildArgValue2 = "VALUE_2" - testBuildContextLocator = testModulePackageId - testContainerImageLocator = "github.com/kurtosistech/test-package/Dockerfile" - testContainerImageLocatorWithBuildFile = "github.com/kurtosistech/test-package/foo.Dockerfile" - testOnDiskContextDirPath = "kurtosis-data/test-package" - testOnDiskContainerImagePath = "kurtosis-data/test-package/Dockerfile" - testOnDiskContainerImagePathWithBuildFile = "kurtosis-data/test-package/foo.Dockerfile" + testModulePackageId = "github.com/kurtosistech/test-package" + testModulePackageAbsoluteLocator = startosis_packages.NewPackageAbsoluteLocator(testModulePackageId, "") + testModuleMainFileLocator = "github.com/kurtosistech/test-package/main.star" + testModuleFileName = "github.com/kurtosistech/test-package/helpers.star" + testModuleRelativeLocator = "./helpers.star" + + testModuleAbsoluteLocator = startosis_packages.NewPackageAbsoluteLocator(testModuleFileName, "") + + emptyAbsoluteLocator = startosis_packages.NewPackageAbsoluteLocator("", "") + + testContainerImageName = "kurtosistech/example-datastore-server" + testBuildContextDir = "./" + testBuildFile = "foo.Dockerfile" + testTargetStage = "builder" + testBuildArgName1 = "BUILD_ARG_1" + testBuildArgValue1 = "VALUE_1" + testBuildArgName2 = "BUILD_ARG_2" + testBuildArgValue2 = "VALUE_2" + testBuildContextAbsoluteLocator = startosis_packages.NewPackageAbsoluteLocator(testModulePackageId, "") + testContainerImageAbsoluteLocator = startosis_packages.NewPackageAbsoluteLocator("github.com/kurtosistech/test-package/Dockerfile", "") + testContainerImageAbsoluteLocatorWithBuildFile = startosis_packages.NewPackageAbsoluteLocator("github.com/kurtosistech/test-package/foo.Dockerfile", "") + testOnDiskContextDirPath = "kurtosis-data/test-package" + testOnDiskContainerImagePath = "kurtosis-data/test-package/Dockerfile" + testOnDiskContainerImagePathWithBuildFile = "kurtosis-data/test-package/foo.Dockerfile" testNixContextDir = "./" testNixImageName = "test-image" @@ -48,7 +54,7 @@ var ( testOnDiskNixContextDirPath = "kurtosis-data/test-package" testOnDiskNixFlakePath = "kurtosis-data/test-package/server/app/flake.nix" testOnDiskNixFlakeDir = "kurtosis-data/test-package/server/app" - testNixFlakeLocator = "github.com/kurtosistech/test-package/server/app/flake.nix" + testNixFlakeAbsoluteLocator = startosis_packages.NewPackageAbsoluteLocator("github.com/kurtosistech/test-package/server/app/flake.nix", "") testRegistryAddr = "http://registry.test.io" testRegistryUsername = "kurtosis" diff --git a/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/image_build_spec.go b/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/image_build_spec.go index c494981c53..f4303e1a26 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/image_build_spec.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/image_build_spec.go @@ -232,11 +232,13 @@ func getOnDiskImageBuildSpecPaths( } // get on disk directory path of Dockerfile - containerImageAbsoluteLocator := path.Join(contextDirAbsoluteLocator, defaultContainerImageFileName) + containerImageAbsoluteLocatorStr := path.Join(contextDirAbsoluteLocator.GetLocator(), defaultContainerImageFileName) if buildFile != "" { - containerImageAbsoluteLocator = path.Join(contextDirAbsoluteLocator, buildFile) + containerImageAbsoluteLocatorStr = path.Join(contextDirAbsoluteLocator.GetLocator(), buildFile) } + containerImageAbsoluteLocator := startosis_packages.NewPackageAbsoluteLocator(containerImageAbsoluteLocatorStr, contextDirAbsoluteLocator.GetTagBranchOrCommit()) + containerImagePathOnDisk, interpretationErr := packageContentProvider.GetOnDiskAbsolutePackageFilePath(containerImageAbsoluteLocator) if interpretationErr != nil { return "", "", interpretationErr diff --git a/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/nix_build_spec.go b/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/nix_build_spec.go index 5e82f085e5..7e994f0fa2 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/nix_build_spec.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/nix_build_spec.go @@ -214,7 +214,9 @@ func getOnDiskNixBuildSpecPaths( } // get on disk directory path of Dockerfile - flakeNixAbsoluteLocator := path.Join(contextDirAbsoluteLocator, flakeLocationDir, defaultNixFlakeFile) + flakeNixAbsoluteLocatorStr := path.Join(contextDirAbsoluteLocator.GetLocator(), flakeLocationDir, defaultNixFlakeFile) + + flakeNixAbsoluteLocator := startosis_packages.NewPackageAbsoluteLocator(flakeNixAbsoluteLocatorStr, contextDirAbsoluteLocator.GetTagBranchOrCommit()) flakeNixPathOnDisk, interpretationErr := packageContentProvider.GetOnDiskAbsolutePackageFilePath(flakeNixAbsoluteLocator) if interpretationErr != nil { diff --git a/core/server/api_container/server/startosis_engine/startosis_packages/absolute_locator.go b/core/server/api_container/server/startosis_engine/startosis_packages/absolute_locator.go new file mode 100644 index 0000000000..9a83900fa1 --- /dev/null +++ b/core/server/api_container/server/startosis_engine/startosis_packages/absolute_locator.go @@ -0,0 +1,35 @@ +package startosis_packages + +import "fmt" + +const ( + defaultMainBranch = "" +) + +// PackageAbsoluteLocator represents the absolute locator for a file in a Kurtosis package +type PackageAbsoluteLocator struct { + // it is the file locator value + locator string + // indicates if the absolute locator correspond to a specific tag, branch or commit + // the zero value (empty string) correspond to the main (or master) branch + tagBranchOrCommit string +} + +func NewPackageAbsoluteLocator(locator string, tagBranchOrCommit string) *PackageAbsoluteLocator { + return &PackageAbsoluteLocator{locator: locator, tagBranchOrCommit: tagBranchOrCommit} +} + +func (packageAbsoluteLocator *PackageAbsoluteLocator) GetLocator() string { + return packageAbsoluteLocator.locator +} + +func (packageAbsoluteLocator *PackageAbsoluteLocator) GetTagBranchOrCommit() string { + return packageAbsoluteLocator.tagBranchOrCommit +} + +func (packageAbsoluteLocator *PackageAbsoluteLocator) GetGitURL() string { + if packageAbsoluteLocator.tagBranchOrCommit == defaultMainBranch { + return packageAbsoluteLocator.locator + } + return fmt.Sprintf("%s@%s", packageAbsoluteLocator.locator, packageAbsoluteLocator.tagBranchOrCommit) +} diff --git a/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/git_package_content_provider.go b/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/git_package_content_provider.go index a3cf9e0cfc..99b2d9796f 100644 --- a/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/git_package_content_provider.go +++ b/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/git_package_content_provider.go @@ -11,6 +11,7 @@ import ( "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/user_support_constants" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_constants" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_errors" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_packages" "github.com/kurtosis-tech/kurtosis/core/server/commons/yaml_parser" "github.com/kurtosis-tech/stacktrace" "github.com/mholt/archiver" @@ -45,6 +46,8 @@ const ( osPathSeparatorString = string(os.PathSeparator) onlyOneReplace = 1 + + defaultMainBranch = "" ) type GitPackageContentProvider struct { @@ -96,21 +99,23 @@ func (provider *GitPackageContentProvider) GetKurtosisYaml(packageAbsolutePathOn return kurtosisYaml, nil } -func (provider *GitPackageContentProvider) GetOnDiskAbsolutePath(repositoryPathURL string) (string, *startosis_errors.InterpretationError) { +func (provider *GitPackageContentProvider) GetOnDiskAbsolutePath(absoluteModuleLocator *startosis_packages.PackageAbsoluteLocator) (string, *startosis_errors.InterpretationError) { shouldOnlyAcceptsPackagePath := false - return provider.getOnDiskAbsolutePath(repositoryPathURL, shouldOnlyAcceptsPackagePath) + return provider.getOnDiskAbsolutePath(absoluteModuleLocator, shouldOnlyAcceptsPackagePath) } -func (provider *GitPackageContentProvider) GetOnDiskAbsolutePackageFilePath(repositoryPathURL string) (string, *startosis_errors.InterpretationError) { +func (provider *GitPackageContentProvider) GetOnDiskAbsolutePackageFilePath(absoluteLocator *startosis_packages.PackageAbsoluteLocator) (string, *startosis_errors.InterpretationError) { shouldOnlyAcceptsPackagePath := true - return provider.getOnDiskAbsolutePath(repositoryPathURL, shouldOnlyAcceptsPackagePath) + return provider.getOnDiskAbsolutePath(absoluteLocator, shouldOnlyAcceptsPackagePath) } -func (provider *GitPackageContentProvider) getOnDiskAbsolutePath(repositoryPathURL string, shouldOnlyAcceptsPackageFilePath bool) (string, *startosis_errors.InterpretationError) { +func (provider *GitPackageContentProvider) getOnDiskAbsolutePath(absoluteLocator *startosis_packages.PackageAbsoluteLocator, shouldOnlyAcceptsPackageFilePath bool) (string, *startosis_errors.InterpretationError) { + repositoryPathURL := absoluteLocator.GetGitURL() parsedURL, err := shared_utils.ParseGitURL(repositoryPathURL) if err != nil { return "", startosis_errors.WrapWithInterpretationError(err, "An error occurred parsing Git URL for absolute file locator '%s'", repositoryPathURL) } + if shouldOnlyAcceptsPackageFilePath && parsedURL.GetRelativeFilePath() == "" { return "", startosis_errors.NewInterpretationError("The path '%v' needs to point to a specific file but it didn't. Users can only read or import specific files and not entire packages.", repositoryPathURL) } @@ -155,8 +160,8 @@ func (provider *GitPackageContentProvider) getOnDiskAbsolutePath(repositoryPathU return pathToFileOnDisk, nil } -func (provider *GitPackageContentProvider) GetModuleContents(fileInsideModuleUrl string) (string, *startosis_errors.InterpretationError) { - pathToFile, interpretationError := provider.GetOnDiskAbsolutePackageFilePath(fileInsideModuleUrl) +func (provider *GitPackageContentProvider) GetModuleContents(absoluteLocator *startosis_packages.PackageAbsoluteLocator) (string, *startosis_errors.InterpretationError) { + pathToFile, interpretationError := provider.GetOnDiskAbsolutePackageFilePath(absoluteLocator) if interpretationError != nil { return "", interpretationError } @@ -164,7 +169,7 @@ func (provider *GitPackageContentProvider) GetModuleContents(fileInsideModuleUrl // Load the file content from its absolute path contents, err := os.ReadFile(pathToFile) if err != nil { - return "", startosis_errors.WrapWithInterpretationError(err, "Loading module content for module '%s' failed. An error occurred in reading contents of the file '%v'", fileInsideModuleUrl, pathToFile) + return "", startosis_errors.WrapWithInterpretationError(err, "Loading module content for module '%s' failed. An error occurred in reading contents of the file '%v'", absoluteLocator.GetLocator(), pathToFile) } return string(contents), nil @@ -230,28 +235,30 @@ func (provider *GitPackageContentProvider) GetAbsoluteLocator( sourceModuleLocator string, relativeOrAbsoluteLocator string, packageReplaceOptions map[string]string, -) (string, *startosis_errors.InterpretationError) { - var absoluteLocator string +) (*startosis_packages.PackageAbsoluteLocator, *startosis_errors.InterpretationError) { + var absoluteLocatorStr string if shouldBlockAbsoluteLocatorBecauseIsInTheSameSourceModuleLocatorPackage(relativeOrAbsoluteLocator, sourceModuleLocator, packageId) { - return "", startosis_errors.NewInterpretationError("Locator '%s' is referencing a file within the same package using absolute import syntax, but only relative import syntax (path starting with '/' or '.') is allowed for within-package imports", relativeOrAbsoluteLocator) + return nil, startosis_errors.NewInterpretationError("Locator '%s' is referencing a file within the same package using absolute import syntax, but only relative import syntax (path starting with '/' or '.') is allowed for within-package imports", relativeOrAbsoluteLocator) } // maybe it's not a relative url in which case we return the url _, errorParsingUrl := shared_utils.ParseGitURL(relativeOrAbsoluteLocator) if errorParsingUrl == nil { // Parsing succeeded, meaning this is already an absolute locator and no relative -> absolute translation is needed - absoluteLocator = relativeOrAbsoluteLocator + absoluteLocatorStr = relativeOrAbsoluteLocator } else { // Parsing did not succeed, meaning this is a relative locator sourceModuleParsedGitUrl, errorParsingPackageId := shared_utils.ParseGitURL(sourceModuleLocator) if errorParsingPackageId != nil { - return "", startosis_errors.NewInterpretationError("Source module locator '%v' isn't a valid locator; relative URLs don't work with standalone scripts", sourceModuleLocator) + return nil, startosis_errors.NewInterpretationError("Source module locator '%v' isn't a valid locator; relative URLs don't work with standalone scripts", sourceModuleLocator) } - absoluteLocator = sourceModuleParsedGitUrl.GetAbsoluteLocatorRelativeToThisURL(relativeOrAbsoluteLocator) + absoluteLocatorStr = sourceModuleParsedGitUrl.GetAbsoluteLocatorRelativeToThisURL(relativeOrAbsoluteLocator) } + absoluteLocator := startosis_packages.NewPackageAbsoluteLocator(absoluteLocatorStr, defaultMainBranch) + replacedAbsoluteLocator := replaceAbsoluteLocator(absoluteLocator, packageReplaceOptions) return replacedAbsoluteLocator, nil diff --git a/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/git_package_content_provider_test.go b/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/git_package_content_provider_test.go index 9d6e367497..e5307b8ac2 100644 --- a/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/git_package_content_provider_test.go +++ b/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/git_package_content_provider_test.go @@ -6,6 +6,7 @@ import ( "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/database_accessors/enclave_db" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_constants" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_errors" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_packages" "github.com/kurtosis-tech/kurtosis/core/server/commons/yaml_parser" "github.com/kurtosis-tech/stacktrace" "github.com/stretchr/testify/require" @@ -43,7 +44,10 @@ func TestGitPackageProvider_SucceedsForValidPackage(t *testing.T) { provider := NewGitPackageContentProvider(packageDir, packageTmpDir, githubAuthTokenFilePath.Name(), nil) sampleStartosisModule := "github.com/kurtosis-tech/sample-startosis-load/sample.star" - contents, err := provider.GetModuleContents(sampleStartosisModule) + + sampleStartosisModuleAbsoluteLocator := startosis_packages.NewPackageAbsoluteLocator(sampleStartosisModule, defaultMainBranch) + + contents, err := provider.GetModuleContents(sampleStartosisModuleAbsoluteLocator) require.Nil(t, err) require.Equal(t, "a = \"World!\"\n", contents) } @@ -64,7 +68,10 @@ func TestGitPackageProvider_SucceedsForValidPackageWithExplicitMasterSet(t *test provider := NewGitPackageContentProvider(packageDir, packageTmpDir, githubAuthTokenFilePath.Name(), nil) sampleStartosisModule := "github.com/kurtosis-tech/sample-startosis-load/sample.star@main" - contents, err := provider.GetModuleContents(sampleStartosisModule) + + sampleStartosisModuleAbsoluteLocator := startosis_packages.NewPackageAbsoluteLocator(sampleStartosisModule, defaultMainBranch) + + contents, err := provider.GetModuleContents(sampleStartosisModuleAbsoluteLocator) require.Nil(t, err) require.Equal(t, "a = \"World!\"\n", contents) } @@ -85,7 +92,10 @@ func TestGitPackageProvider_SucceedsForValidPackageWithBranch(t *testing.T) { provider := NewGitPackageContentProvider(packageDir, packageTmpDir, githubAuthTokenFilePath.Name(), nil) sampleStartosisModule := "github.com/kurtosis-tech/sample-startosis-load/sample.star@test-branch" - contents, err := provider.GetModuleContents(sampleStartosisModule) + + sampleStartosisModuleAbsoluteLocator := startosis_packages.NewPackageAbsoluteLocator(sampleStartosisModule, defaultMainBranch) + + contents, err := provider.GetModuleContents(sampleStartosisModuleAbsoluteLocator) require.Nil(t, err) require.Equal(t, "a = \"Maybe!\"\n", contents) } @@ -106,7 +116,10 @@ func TestGitPackageProvider_FailsForInvalidBranch(t *testing.T) { provider := NewGitPackageContentProvider(packageDir, packageTmpDir, githubAuthTokenFilePath.Name(), nil) sampleStartosisModule := "github.com/kurtosis-tech/sample-startosis-load/sample.star@non-existent-branch" - _, err = provider.GetModuleContents(sampleStartosisModule) + + sampleStartosisModuleAbsoluteLocator := startosis_packages.NewPackageAbsoluteLocator(sampleStartosisModule, defaultMainBranch) + + _, err = provider.GetModuleContents(sampleStartosisModuleAbsoluteLocator) require.NotNil(t, err) } @@ -126,7 +139,10 @@ func TestGitPackageProvider_SucceedsForValidPackageWithTag(t *testing.T) { provider := NewGitPackageContentProvider(packageDir, packageTmpDir, githubAuthTokenFilePath.Name(), nil) sampleStartosisModule := "github.com/kurtosis-tech/sample-startosis-load/sample.star@0.1.1" - contents, err := provider.GetModuleContents(sampleStartosisModule) + + sampleStartosisModuleAbsoluteLocator := startosis_packages.NewPackageAbsoluteLocator(sampleStartosisModule, defaultMainBranch) + + contents, err := provider.GetModuleContents(sampleStartosisModuleAbsoluteLocator) require.Nil(t, err) require.Equal(t, "a = \"World!\"\n", contents) } @@ -147,7 +163,10 @@ func TestGitPackageProvider_SucceedsForValidPackageWithCommit(t *testing.T) { provider := NewGitPackageContentProvider(packageDir, packageTmpDir, githubAuthTokenFilePath.Name(), nil) sampleStartosisModule := "github.com/kurtosis-tech/sample-startosis-load/sample.star@ec9062828e1a687a5db7dfa750f754f88119e4c0" - contents, err := provider.GetModuleContents(sampleStartosisModule) + + sampleStartosisModuleAbsoluteLocator := startosis_packages.NewPackageAbsoluteLocator(sampleStartosisModule, defaultMainBranch) + + contents, err := provider.GetModuleContents(sampleStartosisModuleAbsoluteLocator) require.Nil(t, err) require.Equal(t, "a = \"World!\"\n", contents) } @@ -168,7 +187,10 @@ func TestGitPackageProvider_SucceedsForValidPackageWithCommitOnABranch(t *testin provider := NewGitPackageContentProvider(packageDir, packageTmpDir, githubAuthTokenFilePath.Name(), nil) sampleStartosisModule := "github.com/kurtosis-tech/sample-startosis-load/sample.star@df88baf51caffbe7e8f66c0e54715f680f4482b2" - contents, err := provider.GetModuleContents(sampleStartosisModule) + + sampleStartosisModuleAbsoluteLocator := startosis_packages.NewPackageAbsoluteLocator(sampleStartosisModule, defaultMainBranch) + + contents, err := provider.GetModuleContents(sampleStartosisModuleAbsoluteLocator) require.Nil(t, err) require.Equal(t, "a = \"Maybe!\"\n", contents) } @@ -190,7 +212,10 @@ func TestGitPackageProvider_SucceedsForNonStarlarkFile(t *testing.T) { // TODO replace this with something local or static sampleStarlarkPackage := "github.com/kurtosis-tech/prometheus-package/static-files/prometheus.yml.tmpl" - contents, err := provider.GetModuleContents(sampleStarlarkPackage) + + sampleStarlarkPackageAbsoluteLocator := startosis_packages.NewPackageAbsoluteLocator(sampleStarlarkPackage, defaultMainBranch) + + contents, err := provider.GetModuleContents(sampleStarlarkPackageAbsoluteLocator) require.Nil(t, err) require.NotEmpty(t, contents) } @@ -211,10 +236,87 @@ func TestGitPackageProvider_FailsForNonExistentPackage(t *testing.T) { provider := NewGitPackageContentProvider(oackageDir, packageTmpDir, githubAuthTokenFilePath.Name(), nil) nonExistentModulePath := "github.com/kurtosis-tech/non-existent-startosis-load/sample.star" - _, err = provider.GetModuleContents(nonExistentModulePath) + nonExistentModuleAbsoluteLocator := startosis_packages.NewPackageAbsoluteLocator(nonExistentModulePath, defaultMainBranch) + + _, err = provider.GetModuleContents(nonExistentModuleAbsoluteLocator) require.NotNil(t, err) } +func TestGitPackageProvider_GetContentFromAbsoluteLocatorWithCommit(t *testing.T) { + packageDir, err := os.MkdirTemp("", packagesDirRelPath) + require.Nil(t, err) + defer os.RemoveAll(packageDir) + packageTmpDir, err := os.MkdirTemp("", repositoriesTmpDirRelPath) + require.Nil(t, err) + defer os.RemoveAll(packageTmpDir) + githubAuthDir, err := os.MkdirTemp("", githubAuthDirRelPath) + require.Nil(t, err) + githubAuthTokenFilePath, err := os.CreateTemp(githubAuthDir, githubAuthTokenFilename) + require.Nil(t, err) + defer os.RemoveAll(githubAuthDir) + + provider := NewGitPackageContentProvider(packageDir, packageTmpDir, githubAuthTokenFilePath.Name(), nil) + + absoluteLocatorStr := "github.com/kurtosis-tech/ethereum-package/src/package_io/input_parser.star" + commitHash := "da55be84861e93ce777076e545abee35ff2d51ce" + + absoluteLocatorWithCommit := startosis_packages.NewPackageAbsoluteLocator(absoluteLocatorStr, commitHash) + + contents, err := provider.GetModuleContents(absoluteLocatorWithCommit) + require.Nil(t, err) + require.NotEmpty(t, contents) +} + +func TestGitPackageProvider_GetContentFromAbsoluteLocatorWithCommitComparedWithMainBranch(t *testing.T) { + packageDir, err := os.MkdirTemp("", packagesDirRelPath) + require.Nil(t, err) + defer os.RemoveAll(packageDir) + packageTmpDir, err := os.MkdirTemp("", repositoriesTmpDirRelPath) + require.Nil(t, err) + defer os.RemoveAll(packageTmpDir) + githubAuthDir, err := os.MkdirTemp("", githubAuthDirRelPath) + require.Nil(t, err) + githubAuthTokenFilePath, err := os.CreateTemp(githubAuthDir, githubAuthTokenFilename) + require.Nil(t, err) + defer os.RemoveAll(githubAuthDir) + + provider := NewGitPackageContentProvider(packageDir, packageTmpDir, githubAuthTokenFilePath.Name(), nil) + + absoluteLocatorStr := "github.com/kurtosis-tech/another-sample-dependency-package/directory/internal-module.star" + commitHashInMainBranch := "" + + absoluteLocatorWithCommitInMainBranch := startosis_packages.NewPackageAbsoluteLocator(absoluteLocatorStr, commitHashInMainBranch) + + contentFromMainBranch, err := provider.GetModuleContents(absoluteLocatorWithCommitInMainBranch) + require.Nil(t, err) + require.NotEmpty(t, contentFromMainBranch) + + packageDir, err = os.MkdirTemp("", packagesDirRelPath) + require.Nil(t, err) + defer os.RemoveAll(packageDir) + packageTmpDir, err = os.MkdirTemp("", repositoriesTmpDirRelPath) + require.Nil(t, err) + defer os.RemoveAll(packageTmpDir) + githubAuthDir, err = os.MkdirTemp("", githubAuthDirRelPath) + require.Nil(t, err) + githubAuthTokenFilePath, err = os.CreateTemp(githubAuthDir, githubAuthTokenFilename) + require.Nil(t, err) + defer os.RemoveAll(githubAuthDir) + + provider2 := NewGitPackageContentProvider(packageDir, packageTmpDir, githubAuthTokenFilePath.Name(), nil) + + commitHashInAnotherBranch := "f610049f1f9174bce871431af7d5d35cb6bfd76d" + + absoluteLocatorWithCommitInAnotherBranch := startosis_packages.NewPackageAbsoluteLocator(absoluteLocatorStr, commitHashInAnotherBranch) + + contentFromAnotherBranch, err := provider2.GetModuleContents(absoluteLocatorWithCommitInAnotherBranch) + require.Nil(t, err) + require.NotEmpty(t, contentFromAnotherBranch) + + require.NotEqual(t, contentFromMainBranch, contentFromAnotherBranch) + +} + func TestGetAbsolutePathOnDisk_WorksForPureDirectories(t *testing.T) { packageDir, err := os.MkdirTemp("", packagesDirRelPath) require.Nil(t, err) @@ -231,7 +333,10 @@ func TestGetAbsolutePathOnDisk_WorksForPureDirectories(t *testing.T) { provider := NewGitPackageContentProvider(packageDir, packageTmpDir, githubAuthTokenFilePath.Name(), nil) packagePath := "github.com/kurtosis-tech/datastore-army-package/src/helpers.star" - pathOnDisk, err := provider.getOnDiskAbsolutePath(packagePath, true) + + absoluteLocator := startosis_packages.NewPackageAbsoluteLocator(packagePath, defaultMainBranch) + + pathOnDisk, err := provider.getOnDiskAbsolutePath(absoluteLocator, true) require.Nil(t, err, "This test depends on your internet working and the kurtosis-tech/datastore-army-package existing") require.Equal(t, path.Join(packageDir, "kurtosis-tech", "datastore-army-package", "src/helpers.star"), pathOnDisk) @@ -253,7 +358,10 @@ func TestGetAbsolutePathOnDisk_WorksForNonInMainBranchLocators(t *testing.T) { provider := NewGitPackageContentProvider(packageDir, packageTmpDir, githubAuthTokenFilePath.Name(), nil) absoluteFileLocator := "github.com/kurtosis-tech/sample-dependency-package@test-branch/main.star" - pathOnDisk, err := provider.getOnDiskAbsolutePath(absoluteFileLocator, true) + + absoluteLocator := startosis_packages.NewPackageAbsoluteLocator(absoluteFileLocator, defaultMainBranch) + + pathOnDisk, err := provider.getOnDiskAbsolutePath(absoluteLocator, true) require.Nil(t, err, "This test depends on your internet working and the kurtosis-tech/datastore-army-package existing") require.Equal(t, path.Join(packageDir, "kurtosis-tech", "sample-dependency-package", "main.star"), pathOnDisk) @@ -276,7 +384,10 @@ func TestGetAbsolutePathOnDisk_GenericRepositoryDir(t *testing.T) { provider := NewGitPackageContentProvider(repositoriesDir, repositoriesTmpDir, githubAuthTokenFilePath.Name(), nil) repositoryPathURL := "github.com/kurtosis-tech/minimal-grpc-server/golang/scripts" - pathOnDisk, err := provider.GetOnDiskAbsolutePath(repositoryPathURL) + + absoluteLocator := startosis_packages.NewPackageAbsoluteLocator(repositoryPathURL, defaultMainBranch) + + pathOnDisk, err := provider.GetOnDiskAbsolutePath(absoluteLocator) require.Nil(t, err, "This test depends on your internet working and the kurtosis-tech/minimal-grpc-server existing") require.Equal(t, path.Join(repositoriesDir, "kurtosis-tech", "minimal-grpc-server", "golang", "scripts"), pathOnDisk) @@ -300,7 +411,10 @@ func TestGetAbsolutePathOnDisk_GenericRepositoryFile(t *testing.T) { provider := NewGitPackageContentProvider(repositoriesDir, repositoriesTmpDir, githubAuthTokenFilePath.Name(), nil) repositoryPathURL := "github.com/kurtosis-tech/minimal-grpc-server/golang/scripts/build.sh" - pathOnDisk, err := provider.GetOnDiskAbsolutePath(repositoryPathURL) + + absoluteLocator := startosis_packages.NewPackageAbsoluteLocator(repositoryPathURL, defaultMainBranch) + + pathOnDisk, err := provider.GetOnDiskAbsolutePath(absoluteLocator) require.Nil(t, err, "This test depends on your internet working and the kurtosis-tech/minimal-grpc-server existing") require.Equal(t, path.Join(repositoriesDir, "kurtosis-tech", "minimal-grpc-server", "golang", "scripts", "build.sh"), pathOnDisk) @@ -316,7 +430,7 @@ func TestGetAbsoluteLocator_SucceedsForRelativeFile(t *testing.T) { require.Nil(t, err) expectedAbsoluteLocator := "github.com/kurtosis-tech/avalanche-package/static_files/config.json.tmpl" - require.Equal(t, expectedAbsoluteLocator, absoluteLocator) + require.Equal(t, expectedAbsoluteLocator, absoluteLocator.GetLocator()) parentModuleId2 := "github.com/kurtosis-tech/avalanche-package/src/builder.star" maybeRelativeLocator2 := "/static_files/genesis.json" @@ -324,7 +438,7 @@ func TestGetAbsoluteLocator_SucceedsForRelativeFile(t *testing.T) { expectedAbsoluteLocator2 := "github.com/kurtosis-tech/avalanche-package/static_files/genesis.json" require.Nil(t, err2) - require.Equal(t, expectedAbsoluteLocator2, absoluteLocator2) + require.Equal(t, expectedAbsoluteLocator2, absoluteLocator2.GetLocator()) } func TestGetAbsoluteLocator_RegularReplaceSucceeds(t *testing.T) { @@ -340,10 +454,26 @@ func TestGetAbsoluteLocator_RegularReplaceSucceeds(t *testing.T) { expectedAbsoluteLocator := "github.com/kurtosis-tech/another-sample-dependency-package/main.star" require.Nil(t, err) - require.Equal(t, expectedAbsoluteLocator, absoluteLocator) + require.Equal(t, expectedAbsoluteLocator, absoluteLocator.GetLocator()) } +func TestGetAbsoluteLocator_AnotherPackageWithCommitReplaceSucceeds(t *testing.T) { + provider := NewGitPackageContentProvider("", "", "", nil) + + packageId := "github.com/kurtosis-tech/sample-startosis-load/sample-package" + parentModuleId := "github.com/kurtosis-tech/sample-startosis-load/sample-package/main.star" + maybeRelativeLocator := "github.com/kurtosis-tech/ethereum-package/src/package_io/input_parser.star" + packageReplaceOptions := map[string]string{ + "github.com/kurtosis-tech/ethereum-package": "github.com/kurtosis-tech/ethereum-package@da55be84861e93ce777076e545abee35ff2d51ce", + } + absoluteLocator, err := provider.GetAbsoluteLocator(packageId, parentModuleId, maybeRelativeLocator, packageReplaceOptions) + + expectedAbsoluteLocator := "github.com/kurtosis-tech/ethereum-package/src/package_io/input_parser.star" + require.Nil(t, err) + require.Equal(t, expectedAbsoluteLocator, absoluteLocator.GetLocator()) +} + func TestGetAbsoluteLocator_RootPackageReplaceSucceeds(t *testing.T) { provider := NewGitPackageContentProvider("", "", "", nil) @@ -358,7 +488,7 @@ func TestGetAbsoluteLocator_RootPackageReplaceSucceeds(t *testing.T) { expectedAbsoluteLocator := "github.com/kurtosis-tech/root-package-replaced/main.star" require.Nil(t, err) - require.Equal(t, expectedAbsoluteLocator, absoluteLocator) + require.Equal(t, expectedAbsoluteLocator, absoluteLocator.GetLocator()) } @@ -376,7 +506,7 @@ func TestGetAbsoluteLocator_SubPackageReplaceSucceeds(t *testing.T) { expectedAbsoluteLocator := "github.com/kurtosis-tech/sub-package-replaced/main.star" require.Nil(t, err) - require.Equal(t, expectedAbsoluteLocator, absoluteLocator) + require.Equal(t, expectedAbsoluteLocator, absoluteLocator.GetLocator()) } @@ -393,7 +523,7 @@ func TestGetAbsoluteLocator_ReplacePackageInternalModuleSucceeds(t *testing.T) { expectedAbsoluteLocator := "github.com/kurtosis-tech/root-package-replaced/folder/module.star" require.Nil(t, err) - require.Equal(t, expectedAbsoluteLocator, absoluteLocator) + require.Equal(t, expectedAbsoluteLocator, absoluteLocator.GetLocator()) } func TestGetAbsoluteLocator_NoMainBranchReplaceSucceeds(t *testing.T) { @@ -407,9 +537,9 @@ func TestGetAbsoluteLocator_NoMainBranchReplaceSucceeds(t *testing.T) { } absoluteLocator, err := provider.GetAbsoluteLocator(packageId, parentModuleId, maybeRelativeLocator, packageReplaceOptions) - expectedAbsoluteLocator := "github.com/kurtosis-tech/sample-dependency-package@no-main-branch/main.star" + expectedAbsoluteLocator := "github.com/kurtosis-tech/sample-dependency-package/main.star" require.Nil(t, err) - require.Equal(t, expectedAbsoluteLocator, absoluteLocator) + require.Equal(t, expectedAbsoluteLocator, absoluteLocator.GetLocator()) } func TestGetAbsoluteLocator_ShouldBlockSamePackageAbsoluteLocator(t *testing.T) { diff --git a/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/locators.go b/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/locators.go index 48e168bed5..c5472eb361 100644 --- a/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/locators.go +++ b/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/locators.go @@ -2,6 +2,7 @@ package git_package_content_provider import ( "github.com/kurtosis-tech/kurtosis/api/golang/core/lib/shared_utils" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_packages" "github.com/sirupsen/logrus" "strings" ) @@ -26,12 +27,12 @@ func shouldBlockAbsoluteLocatorBecauseIsInTheSameSourceModuleLocatorPackage(rela return isSourceModuleInRootPackage && isAbsoluteLocatorInRootPackage } -func replaceAbsoluteLocator(absoluteLocator string, packageReplaceOptions map[string]string) string { - if absoluteLocator == "" { +func replaceAbsoluteLocator(absoluteLocator *startosis_packages.PackageAbsoluteLocator, packageReplaceOptions map[string]string) *startosis_packages.PackageAbsoluteLocator { + if absoluteLocator.GetLocator() == "" { return absoluteLocator } - found, packageToBeReplaced, replaceWithPackage := findPackageReplace(absoluteLocator, packageReplaceOptions) + found, packageToBeReplaced, replaceWithPackage, maybeTagBranchOrCommit := findPackageReplace(absoluteLocator, packageReplaceOptions) if found { // we skip if it's a local replace because we will use the same absolute locator @@ -39,20 +40,22 @@ func replaceAbsoluteLocator(absoluteLocator string, packageReplaceOptions map[st if isLocalLocator(replaceWithPackage) { return absoluteLocator } - replacedAbsoluteLocator := strings.Replace(absoluteLocator, packageToBeReplaced, replaceWithPackage, onlyOneReplace) - logrus.Debugf("absoluteLocator '%s' replaced with '%s'", absoluteLocator, replacedAbsoluteLocator) + replacedAbsoluteLocatorStr := strings.Replace(absoluteLocator.GetLocator(), packageToBeReplaced, replaceWithPackage, onlyOneReplace) + replacedAbsoluteLocator := startosis_packages.NewPackageAbsoluteLocator(replacedAbsoluteLocatorStr, maybeTagBranchOrCommit) + logrus.Debugf("absoluteLocator '%s' replaced with '%s' with tag, branch or commit %s", absoluteLocator.GetLocator(), replacedAbsoluteLocator.GetLocator(), replacedAbsoluteLocator.GetTagBranchOrCommit()) + return replacedAbsoluteLocator } return absoluteLocator } -func findPackageReplace(absoluteLocator string, packageReplaceOptions map[string]string) (bool, string, string) { +func findPackageReplace(absoluteLocator *startosis_packages.PackageAbsoluteLocator, packageReplaceOptions map[string]string) (bool, string, string, string) { if len(packageReplaceOptions) == 0 { - return false, "", "" + return false, "", "", "" } - pathToAnalyze := absoluteLocator + pathToAnalyze := absoluteLocator.GetLocator() for { numberSlashes := strings.Count(pathToAnalyze, shared_utils.UrlPathSeparator) @@ -63,14 +66,15 @@ func findPackageReplace(absoluteLocator string, packageReplaceOptions map[string lastIndex := strings.LastIndex(pathToAnalyze, shared_utils.UrlPathSeparator) packageToBeReplaced := pathToAnalyze[:lastIndex] - replaceWithPackage, ok := packageReplaceOptions[packageToBeReplaced] + replacePackageWithMaybeWitBranchOrCommit, ok := packageReplaceOptions[packageToBeReplaced] if ok { + replaceWithPackage, maybeTagBranchOrCommit := shared_utils.ParseOutTagBranchOrCommit(replacePackageWithMaybeWitBranchOrCommit) logrus.Debugf("dependency replace found for '%s', package '%s' will be replaced with '%s'", absoluteLocator, packageToBeReplaced, replaceWithPackage) - return true, packageToBeReplaced, replaceWithPackage + return true, packageToBeReplaced, replaceWithPackage, maybeTagBranchOrCommit } pathToAnalyze = packageToBeReplaced } - return false, "", "" + return false, "", "", "" } diff --git a/core/server/api_container/server/startosis_engine/startosis_packages/mock_package_content_provider.go b/core/server/api_container/server/startosis_engine/startosis_packages/mock_package_content_provider.go index cab172daaf..9748c3f275 100644 --- a/core/server/api_container/server/startosis_engine/startosis_packages/mock_package_content_provider.go +++ b/core/server/api_container/server/startosis_engine/startosis_packages/mock_package_content_provider.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.29.0. DO NOT EDIT. +// Code generated by mockery v2.36.0. DO NOT EDIT. package startosis_packages @@ -122,18 +122,20 @@ func (_c *MockPackageContentProvider_CloneReplacedPackagesIfNeeded_Call) RunAndR } // GetAbsoluteLocator provides a mock function with given fields: packageId, locatorOfModuleInWhichThisBuiltInIsBeingCalled, relativeOrAbsoluteLocator, packageReplaceOptions -func (_m *MockPackageContentProvider) GetAbsoluteLocator(packageId string, locatorOfModuleInWhichThisBuiltInIsBeingCalled string, relativeOrAbsoluteLocator string, packageReplaceOptions map[string]string) (string, *startosis_errors.InterpretationError) { +func (_m *MockPackageContentProvider) GetAbsoluteLocator(packageId string, locatorOfModuleInWhichThisBuiltInIsBeingCalled string, relativeOrAbsoluteLocator string, packageReplaceOptions map[string]string) (*PackageAbsoluteLocator, *startosis_errors.InterpretationError) { ret := _m.Called(packageId, locatorOfModuleInWhichThisBuiltInIsBeingCalled, relativeOrAbsoluteLocator, packageReplaceOptions) - var r0 string + var r0 *PackageAbsoluteLocator var r1 *startosis_errors.InterpretationError - if rf, ok := ret.Get(0).(func(string, string, string, map[string]string) (string, *startosis_errors.InterpretationError)); ok { + if rf, ok := ret.Get(0).(func(string, string, string, map[string]string) (*PackageAbsoluteLocator, *startosis_errors.InterpretationError)); ok { return rf(packageId, locatorOfModuleInWhichThisBuiltInIsBeingCalled, relativeOrAbsoluteLocator, packageReplaceOptions) } - if rf, ok := ret.Get(0).(func(string, string, string, map[string]string) string); ok { + if rf, ok := ret.Get(0).(func(string, string, string, map[string]string) *PackageAbsoluteLocator); ok { r0 = rf(packageId, locatorOfModuleInWhichThisBuiltInIsBeingCalled, relativeOrAbsoluteLocator, packageReplaceOptions) } else { - r0 = ret.Get(0).(string) + if ret.Get(0) != nil { + r0 = ret.Get(0).(*PackageAbsoluteLocator) + } } if rf, ok := ret.Get(1).(func(string, string, string, map[string]string) *startosis_errors.InterpretationError); ok { @@ -168,12 +170,12 @@ func (_c *MockPackageContentProvider_GetAbsoluteLocator_Call) Run(run func(packa return _c } -func (_c *MockPackageContentProvider_GetAbsoluteLocator_Call) Return(_a0 string, _a1 *startosis_errors.InterpretationError) *MockPackageContentProvider_GetAbsoluteLocator_Call { +func (_c *MockPackageContentProvider_GetAbsoluteLocator_Call) Return(_a0 *PackageAbsoluteLocator, _a1 *startosis_errors.InterpretationError) *MockPackageContentProvider_GetAbsoluteLocator_Call { _c.Call.Return(_a0, _a1) return _c } -func (_c *MockPackageContentProvider_GetAbsoluteLocator_Call) RunAndReturn(run func(string, string, string, map[string]string) (string, *startosis_errors.InterpretationError)) *MockPackageContentProvider_GetAbsoluteLocator_Call { +func (_c *MockPackageContentProvider_GetAbsoluteLocator_Call) RunAndReturn(run func(string, string, string, map[string]string) (*PackageAbsoluteLocator, *startosis_errors.InterpretationError)) *MockPackageContentProvider_GetAbsoluteLocator_Call { _c.Call.Return(run) return _c } @@ -234,23 +236,23 @@ func (_c *MockPackageContentProvider_GetKurtosisYaml_Call) RunAndReturn(run func return _c } -// GetModuleContents provides a mock function with given fields: fileInsidePackageUrl -func (_m *MockPackageContentProvider) GetModuleContents(fileInsidePackageUrl string) (string, *startosis_errors.InterpretationError) { - ret := _m.Called(fileInsidePackageUrl) +// GetModuleContents provides a mock function with given fields: absoluteModuleLocator +func (_m *MockPackageContentProvider) GetModuleContents(absoluteModuleLocator *PackageAbsoluteLocator) (string, *startosis_errors.InterpretationError) { + ret := _m.Called(absoluteModuleLocator) var r0 string var r1 *startosis_errors.InterpretationError - if rf, ok := ret.Get(0).(func(string) (string, *startosis_errors.InterpretationError)); ok { - return rf(fileInsidePackageUrl) + if rf, ok := ret.Get(0).(func(*PackageAbsoluteLocator) (string, *startosis_errors.InterpretationError)); ok { + return rf(absoluteModuleLocator) } - if rf, ok := ret.Get(0).(func(string) string); ok { - r0 = rf(fileInsidePackageUrl) + if rf, ok := ret.Get(0).(func(*PackageAbsoluteLocator) string); ok { + r0 = rf(absoluteModuleLocator) } else { r0 = ret.Get(0).(string) } - if rf, ok := ret.Get(1).(func(string) *startosis_errors.InterpretationError); ok { - r1 = rf(fileInsidePackageUrl) + if rf, ok := ret.Get(1).(func(*PackageAbsoluteLocator) *startosis_errors.InterpretationError); ok { + r1 = rf(absoluteModuleLocator) } else { if ret.Get(1) != nil { r1 = ret.Get(1).(*startosis_errors.InterpretationError) @@ -266,14 +268,14 @@ type MockPackageContentProvider_GetModuleContents_Call struct { } // GetModuleContents is a helper method to define mock.On call -// - fileInsidePackageUrl string -func (_e *MockPackageContentProvider_Expecter) GetModuleContents(fileInsidePackageUrl interface{}) *MockPackageContentProvider_GetModuleContents_Call { - return &MockPackageContentProvider_GetModuleContents_Call{Call: _e.mock.On("GetModuleContents", fileInsidePackageUrl)} +// - absoluteModuleLocator *PackageAbsoluteLocator +func (_e *MockPackageContentProvider_Expecter) GetModuleContents(absoluteModuleLocator interface{}) *MockPackageContentProvider_GetModuleContents_Call { + return &MockPackageContentProvider_GetModuleContents_Call{Call: _e.mock.On("GetModuleContents", absoluteModuleLocator)} } -func (_c *MockPackageContentProvider_GetModuleContents_Call) Run(run func(fileInsidePackageUrl string)) *MockPackageContentProvider_GetModuleContents_Call { +func (_c *MockPackageContentProvider_GetModuleContents_Call) Run(run func(absoluteModuleLocator *PackageAbsoluteLocator)) *MockPackageContentProvider_GetModuleContents_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(string)) + run(args[0].(*PackageAbsoluteLocator)) }) return _c } @@ -283,28 +285,28 @@ func (_c *MockPackageContentProvider_GetModuleContents_Call) Return(_a0 string, return _c } -func (_c *MockPackageContentProvider_GetModuleContents_Call) RunAndReturn(run func(string) (string, *startosis_errors.InterpretationError)) *MockPackageContentProvider_GetModuleContents_Call { +func (_c *MockPackageContentProvider_GetModuleContents_Call) RunAndReturn(run func(*PackageAbsoluteLocator) (string, *startosis_errors.InterpretationError)) *MockPackageContentProvider_GetModuleContents_Call { _c.Call.Return(run) return _c } -// GetOnDiskAbsolutePackageFilePath provides a mock function with given fields: fileInsidePackageUrl -func (_m *MockPackageContentProvider) GetOnDiskAbsolutePackageFilePath(fileInsidePackageUrl string) (string, *startosis_errors.InterpretationError) { - ret := _m.Called(fileInsidePackageUrl) +// GetOnDiskAbsolutePackageFilePath provides a mock function with given fields: absoluteModuleLocator +func (_m *MockPackageContentProvider) GetOnDiskAbsolutePackageFilePath(absoluteModuleLocator *PackageAbsoluteLocator) (string, *startosis_errors.InterpretationError) { + ret := _m.Called(absoluteModuleLocator) var r0 string var r1 *startosis_errors.InterpretationError - if rf, ok := ret.Get(0).(func(string) (string, *startosis_errors.InterpretationError)); ok { - return rf(fileInsidePackageUrl) + if rf, ok := ret.Get(0).(func(*PackageAbsoluteLocator) (string, *startosis_errors.InterpretationError)); ok { + return rf(absoluteModuleLocator) } - if rf, ok := ret.Get(0).(func(string) string); ok { - r0 = rf(fileInsidePackageUrl) + if rf, ok := ret.Get(0).(func(*PackageAbsoluteLocator) string); ok { + r0 = rf(absoluteModuleLocator) } else { r0 = ret.Get(0).(string) } - if rf, ok := ret.Get(1).(func(string) *startosis_errors.InterpretationError); ok { - r1 = rf(fileInsidePackageUrl) + if rf, ok := ret.Get(1).(func(*PackageAbsoluteLocator) *startosis_errors.InterpretationError); ok { + r1 = rf(absoluteModuleLocator) } else { if ret.Get(1) != nil { r1 = ret.Get(1).(*startosis_errors.InterpretationError) @@ -320,14 +322,14 @@ type MockPackageContentProvider_GetOnDiskAbsolutePackageFilePath_Call struct { } // GetOnDiskAbsolutePackageFilePath is a helper method to define mock.On call -// - fileInsidePackageUrl string -func (_e *MockPackageContentProvider_Expecter) GetOnDiskAbsolutePackageFilePath(fileInsidePackageUrl interface{}) *MockPackageContentProvider_GetOnDiskAbsolutePackageFilePath_Call { - return &MockPackageContentProvider_GetOnDiskAbsolutePackageFilePath_Call{Call: _e.mock.On("GetOnDiskAbsolutePackageFilePath", fileInsidePackageUrl)} +// - absoluteModuleLocator *PackageAbsoluteLocator +func (_e *MockPackageContentProvider_Expecter) GetOnDiskAbsolutePackageFilePath(absoluteModuleLocator interface{}) *MockPackageContentProvider_GetOnDiskAbsolutePackageFilePath_Call { + return &MockPackageContentProvider_GetOnDiskAbsolutePackageFilePath_Call{Call: _e.mock.On("GetOnDiskAbsolutePackageFilePath", absoluteModuleLocator)} } -func (_c *MockPackageContentProvider_GetOnDiskAbsolutePackageFilePath_Call) Run(run func(fileInsidePackageUrl string)) *MockPackageContentProvider_GetOnDiskAbsolutePackageFilePath_Call { +func (_c *MockPackageContentProvider_GetOnDiskAbsolutePackageFilePath_Call) Run(run func(absoluteModuleLocator *PackageAbsoluteLocator)) *MockPackageContentProvider_GetOnDiskAbsolutePackageFilePath_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(string)) + run(args[0].(*PackageAbsoluteLocator)) }) return _c } @@ -337,7 +339,7 @@ func (_c *MockPackageContentProvider_GetOnDiskAbsolutePackageFilePath_Call) Retu return _c } -func (_c *MockPackageContentProvider_GetOnDiskAbsolutePackageFilePath_Call) RunAndReturn(run func(string) (string, *startosis_errors.InterpretationError)) *MockPackageContentProvider_GetOnDiskAbsolutePackageFilePath_Call { +func (_c *MockPackageContentProvider_GetOnDiskAbsolutePackageFilePath_Call) RunAndReturn(run func(*PackageAbsoluteLocator) (string, *startosis_errors.InterpretationError)) *MockPackageContentProvider_GetOnDiskAbsolutePackageFilePath_Call { _c.Call.Return(run) return _c } @@ -396,23 +398,23 @@ func (_c *MockPackageContentProvider_GetOnDiskAbsolutePackagePath_Call) RunAndRe return _c } -// GetOnDiskAbsolutePath provides a mock function with given fields: repositoryPathURL -func (_m *MockPackageContentProvider) GetOnDiskAbsolutePath(repositoryPathURL string) (string, *startosis_errors.InterpretationError) { - ret := _m.Called(repositoryPathURL) +// GetOnDiskAbsolutePath provides a mock function with given fields: absoluteModuleLocator +func (_m *MockPackageContentProvider) GetOnDiskAbsolutePath(absoluteModuleLocator *PackageAbsoluteLocator) (string, *startosis_errors.InterpretationError) { + ret := _m.Called(absoluteModuleLocator) var r0 string var r1 *startosis_errors.InterpretationError - if rf, ok := ret.Get(0).(func(string) (string, *startosis_errors.InterpretationError)); ok { - return rf(repositoryPathURL) + if rf, ok := ret.Get(0).(func(*PackageAbsoluteLocator) (string, *startosis_errors.InterpretationError)); ok { + return rf(absoluteModuleLocator) } - if rf, ok := ret.Get(0).(func(string) string); ok { - r0 = rf(repositoryPathURL) + if rf, ok := ret.Get(0).(func(*PackageAbsoluteLocator) string); ok { + r0 = rf(absoluteModuleLocator) } else { r0 = ret.Get(0).(string) } - if rf, ok := ret.Get(1).(func(string) *startosis_errors.InterpretationError); ok { - r1 = rf(repositoryPathURL) + if rf, ok := ret.Get(1).(func(*PackageAbsoluteLocator) *startosis_errors.InterpretationError); ok { + r1 = rf(absoluteModuleLocator) } else { if ret.Get(1) != nil { r1 = ret.Get(1).(*startosis_errors.InterpretationError) @@ -428,14 +430,14 @@ type MockPackageContentProvider_GetOnDiskAbsolutePath_Call struct { } // GetOnDiskAbsolutePath is a helper method to define mock.On call -// - repositoryPathURL string -func (_e *MockPackageContentProvider_Expecter) GetOnDiskAbsolutePath(repositoryPathURL interface{}) *MockPackageContentProvider_GetOnDiskAbsolutePath_Call { - return &MockPackageContentProvider_GetOnDiskAbsolutePath_Call{Call: _e.mock.On("GetOnDiskAbsolutePath", repositoryPathURL)} +// - absoluteModuleLocator *PackageAbsoluteLocator +func (_e *MockPackageContentProvider_Expecter) GetOnDiskAbsolutePath(absoluteModuleLocator interface{}) *MockPackageContentProvider_GetOnDiskAbsolutePath_Call { + return &MockPackageContentProvider_GetOnDiskAbsolutePath_Call{Call: _e.mock.On("GetOnDiskAbsolutePath", absoluteModuleLocator)} } -func (_c *MockPackageContentProvider_GetOnDiskAbsolutePath_Call) Run(run func(repositoryPathURL string)) *MockPackageContentProvider_GetOnDiskAbsolutePath_Call { +func (_c *MockPackageContentProvider_GetOnDiskAbsolutePath_Call) Run(run func(absoluteModuleLocator *PackageAbsoluteLocator)) *MockPackageContentProvider_GetOnDiskAbsolutePath_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(string)) + run(args[0].(*PackageAbsoluteLocator)) }) return _c } @@ -445,7 +447,7 @@ func (_c *MockPackageContentProvider_GetOnDiskAbsolutePath_Call) Return(_a0 stri return _c } -func (_c *MockPackageContentProvider_GetOnDiskAbsolutePath_Call) RunAndReturn(run func(string) (string, *startosis_errors.InterpretationError)) *MockPackageContentProvider_GetOnDiskAbsolutePath_Call { +func (_c *MockPackageContentProvider_GetOnDiskAbsolutePath_Call) RunAndReturn(run func(*PackageAbsoluteLocator) (string, *startosis_errors.InterpretationError)) *MockPackageContentProvider_GetOnDiskAbsolutePath_Call { _c.Call.Return(run) return _c } @@ -506,13 +508,12 @@ func (_c *MockPackageContentProvider_StorePackageContents_Call) RunAndReturn(run return _c } -type mockConstructorTestingTNewMockPackageContentProvider interface { +// NewMockPackageContentProvider creates a new instance of MockPackageContentProvider. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockPackageContentProvider(t interface { mock.TestingT Cleanup(func()) -} - -// NewMockPackageContentProvider creates a new instance of MockPackageContentProvider. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewMockPackageContentProvider(t mockConstructorTestingTNewMockPackageContentProvider) *MockPackageContentProvider { +}) *MockPackageContentProvider { mock := &MockPackageContentProvider{} mock.Mock.Test(t) diff --git a/core/server/api_container/server/startosis_engine/startosis_packages/mock_package_content_provider/mock_package_content_provider.go b/core/server/api_container/server/startosis_engine/startosis_packages/mock_package_content_provider/mock_package_content_provider.go index acd1e7c18d..77916b5f66 100644 --- a/core/server/api_container/server/startosis_engine/startosis_packages/mock_package_content_provider/mock_package_content_provider.go +++ b/core/server/api_container/server/startosis_engine/startosis_packages/mock_package_content_provider/mock_package_content_provider.go @@ -3,6 +3,7 @@ package mock_package_content_provider import ( "github.com/kurtosis-tech/kurtosis/api/golang/core/lib/shared_utils" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_errors" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_packages" "github.com/kurtosis-tech/kurtosis/core/server/commons/yaml_parser" "github.com/kurtosis-tech/stacktrace" "io" @@ -32,24 +33,24 @@ func NewMockPackageContentProvider() *MockPackageContentProvider { } } -func (provider *MockPackageContentProvider) GetOnDiskAbsolutePackageFilePath(fileInsidePackageUrl string) (string, *startosis_errors.InterpretationError) { - absFilePath, found := provider.starlarkPackages[fileInsidePackageUrl] +func (provider *MockPackageContentProvider) GetOnDiskAbsolutePackageFilePath(absoluteModuleLocator *startosis_packages.PackageAbsoluteLocator) (string, *startosis_errors.InterpretationError) { + absFilePath, found := provider.starlarkPackages[absoluteModuleLocator.GetLocator()] if !found { - return "", startosis_errors.NewInterpretationError("Module '%v' not found", fileInsidePackageUrl) + return "", startosis_errors.NewInterpretationError("Module '%v' not found", absoluteModuleLocator.GetLocator()) } if _, err := os.Stat(absFilePath); err != nil { - return "", startosis_errors.NewInterpretationError("Unable to read content of package '%v'", fileInsidePackageUrl) + return "", startosis_errors.NewInterpretationError("Unable to read content of package '%v'", absoluteModuleLocator.GetLocator()) } return absFilePath, nil } -func (provider *MockPackageContentProvider) GetOnDiskAbsolutePath(repositoryPathURL string) (string, *startosis_errors.InterpretationError) { - absFilePath, found := provider.starlarkPackages[repositoryPathURL] +func (provider *MockPackageContentProvider) GetOnDiskAbsolutePath(absoluteModuleLocator *startosis_packages.PackageAbsoluteLocator) (string, *startosis_errors.InterpretationError) { + absFilePath, found := provider.starlarkPackages[absoluteModuleLocator.GetLocator()] if !found { - return "", startosis_errors.NewInterpretationError("Module '%v' not found", repositoryPathURL) + return "", startosis_errors.NewInterpretationError("Module '%v' not found", absoluteModuleLocator.GetLocator()) } if _, err := os.Stat(absFilePath); err != nil { - return "", startosis_errors.NewInterpretationError("Unable to read content of package '%v'", repositoryPathURL) + return "", startosis_errors.NewInterpretationError("Unable to read content of package '%v'", absoluteModuleLocator.GetLocator()) } return absFilePath, nil } @@ -75,27 +76,33 @@ func (provider *MockPackageContentProvider) CloneReplacedPackagesIfNeeded(curren return nil } -func (provider *MockPackageContentProvider) GetModuleContents(fileInsidePackageUrl string) (string, *startosis_errors.InterpretationError) { - absFilePath, found := provider.starlarkPackages[fileInsidePackageUrl] +func (provider *MockPackageContentProvider) GetModuleContents(absoluteModuleLocator *startosis_packages.PackageAbsoluteLocator) (string, *startosis_errors.InterpretationError) { + absFilePath, found := provider.starlarkPackages[absoluteModuleLocator.GetLocator()] if !found { - return "", startosis_errors.NewInterpretationError("Package '%v' not found", fileInsidePackageUrl) + return "", startosis_errors.NewInterpretationError("Package '%v' not found", absoluteModuleLocator.GetLocator()) } fileContent, err := os.ReadFile(absFilePath) if err != nil { - return "", startosis_errors.NewInterpretationError("Unable to read content of package '%v'", fileInsidePackageUrl) + return "", startosis_errors.NewInterpretationError("Unable to read content of package '%v'", absoluteModuleLocator.GetLocator()) } return string(fileContent), nil } -func (provider *MockPackageContentProvider) GetAbsoluteLocator(packageId string, parentModuleId string, relativeOrAbsoluteModulePath string, packageReplaceOptions map[string]string) (string, *startosis_errors.InterpretationError) { +func (provider *MockPackageContentProvider) GetAbsoluteLocator(packageId string, parentModuleId string, relativeOrAbsoluteModulePath string, packageReplaceOptions map[string]string) (*startosis_packages.PackageAbsoluteLocator, *startosis_errors.InterpretationError) { if strings.HasPrefix(relativeOrAbsoluteModulePath, parentModuleId) { - return "", startosis_errors.NewInterpretationError("Cannot use local absolute locators") + return nil, startosis_errors.NewInterpretationError("Cannot use local absolute locators") } + useMainBranch := "" if strings.HasPrefix(relativeOrAbsoluteModulePath, shared_utils.GithubDomainPrefix) { - return relativeOrAbsoluteModulePath, nil + + absoluteLocator := startosis_packages.NewPackageAbsoluteLocator(relativeOrAbsoluteModulePath, useMainBranch) + return absoluteLocator, nil } - return provider.packageId, nil + + absoluteLocator := startosis_packages.NewPackageAbsoluteLocator(provider.packageId, useMainBranch) + + return absoluteLocator, nil } func (provider *MockPackageContentProvider) AddFileContent(packageId string, contents string) error { diff --git a/core/server/api_container/server/startosis_engine/startosis_packages/package_content_provider.go b/core/server/api_container/server/startosis_engine/startosis_packages/package_content_provider.go index d7a0003f76..cbe9ad1518 100644 --- a/core/server/api_container/server/startosis_engine/startosis_packages/package_content_provider.go +++ b/core/server/api_container/server/startosis_engine/startosis_packages/package_content_provider.go @@ -14,14 +14,14 @@ import ( type PackageContentProvider interface { // GetOnDiskAbsolutePackageFilePath returns the absolute file path of a file inside a module. // The corresponding GitHub repo will be cloned if necessary - GetOnDiskAbsolutePackageFilePath(fileInsidePackageUrl string) (string, *startosis_errors.InterpretationError) + GetOnDiskAbsolutePackageFilePath(absoluteModuleLocator *PackageAbsoluteLocator) (string, *startosis_errors.InterpretationError) // GetOnDiskAbsolutePath returns the absolute path (it can be a filer or a folder and, it can be from a package or not) on APIC's disk. // The corresponding GitHub repo will be cloned if necessary - GetOnDiskAbsolutePath(repositoryPathURL string) (string, *startosis_errors.InterpretationError) + GetOnDiskAbsolutePath(absoluteModuleLocator *PackageAbsoluteLocator) (string, *startosis_errors.InterpretationError) // GetModuleContents returns the stringifies content of a file inside a module - GetModuleContents(fileInsidePackageUrl string) (string, *startosis_errors.InterpretationError) + GetModuleContents(absoluteModuleLocator *PackageAbsoluteLocator) (string, *startosis_errors.InterpretationError) // GetOnDiskAbsolutePackagePath returns the absolute folder path containing this package // It throws an error if the package does not exist on disk @@ -36,7 +36,7 @@ type PackageContentProvider interface { // GetAbsoluteLocator does: // 1. if the given locator is relative, translates it to absolute locator using sourceModuleLocator (if it's already absolute, does nothing) // 2. applies any replace rules, if they match the now-absolute locator - GetAbsoluteLocator(packageId string, locatorOfModuleInWhichThisBuiltInIsBeingCalled string, relativeOrAbsoluteLocator string, packageReplaceOptions map[string]string) (string, *startosis_errors.InterpretationError) + GetAbsoluteLocator(packageId string, locatorOfModuleInWhichThisBuiltInIsBeingCalled string, relativeOrAbsoluteLocator string, packageReplaceOptions map[string]string) (*PackageAbsoluteLocator, *startosis_errors.InterpretationError) // GetKurtosisYaml returns the package kurtosis.yml file content GetKurtosisYaml(packageAbsolutePathOnDisk string) (*yaml_parser.KurtosisYaml, *startosis_errors.InterpretationError)