Skip to content
This repository has been archived by the owner on Jul 17, 2023. It is now read-only.

fix: add helm #90

Draft
wants to merge 1 commit into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion docker/example/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -15,5 +15,8 @@ docker_image(
srcs = [":example"],
base_image = ":base",
run_args = "-p 8000:8000",
visibility = ["//k8s/example:all"],
visibility = [
"//k8s/example:all",
"//helm/example:all"
],
)
6 changes: 6 additions & 0 deletions helm/BUILD
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
package(default_visibility=['PUBLIC'])


filegroup(name='helm', srcs=[
'helm.build_defs',
])
23 changes: 23 additions & 0 deletions helm/example/.helmignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*.orig
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/
27 changes: 27 additions & 0 deletions helm/example/BUILD
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
subinclude("//helm")


chart(
name = "chart",
chart_name = "mychart",
repo = "$INTERNAL_SUB_CHART_REPO",
oci_registry = "$CHART_OCI_REGISTRY",
oci_registry_username = "$CHART_OCI_REGISTRY_USERNAME",
oci_registry_password = "$CHART_OCI_REGISTRY_PASSWORD",
version = "$APP_VERSION",
params = {
'EXAMPLE_PARAM': 'test',
'REPOSITORY': "$REPOSITORY",
'APP_VERSION': "$APP_VERSION",
'CHART_OCI_REGISTRY': "$CHART_OCI_REGISTRY",
'EXTERNAL_SUB_CHART_REPO': "$EXTERNAL_SUB_CHART_REPO",
'INTERNAL_SUB_CHART_REPO': "$INTERNAL_SUB_CHART_REPO"

},
containers = [
"//docker/example:image"
],
sub_charts = [

]
)
29 changes: 29 additions & 0 deletions helm/example/Chart.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
apiVersion: v2
name: mychart
description: description

# A chart can be either an 'application' or a 'library' chart.
#
# Application charts are a collection of templates that can be packaged into versioned archives
# to be deployed.
#
# Library charts provide useful utilities or functions for the chart developer. They're included as
# a dependency of application charts to inject those utilities and functions into the rendering
# pipeline. Library charts do not define any templates and therefore cannot be deployed.
type: application

# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: ${APP_VERSION}

# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
appVersion: "${APP_VERSION}"

dependencies:
- name: dex
version: 0.6.3
repository: ${EXTERNAL_SUB_CHART_REPO}
2 changes: 2 additions & 0 deletions helm/example/values.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
image:
repository: docker/example:image
238 changes: 238 additions & 0 deletions helm/helm.build_defs
Original file line number Diff line number Diff line change
@@ -0,0 +1,238 @@
subinclude("//helm/k8s_fork")


def chart(name:str='chart', chart_name:str, version:str='', repo:str='', oci_registry:str, oci_registry_username:str, oci_registry_password:str, containers:list=[], params:dict=None, sub_charts:list=[], labels:list=[], visibility:list=None):
if not repo:
repo = check_config('DEFAULT_DOCKER_REPO', example='repo.com/repo')

sub_charts = [sub_chart for sub_chart in sub_charts if sub_chart]
sub_charts = [canonicalise(sub_chart) for sub_chart in sub_charts]
containers = [canonicalise(container) for container in containers]

chart_tmp_files=filegroup(
name = f'_tmp_files',
srcs = glob(["charts/**/*"], exclude = ["charts/*.gz"]) +
glob(["templates/**/*.yaml"]) +
glob(["templates/**/*.tpl"]) +
[
".helmignore",
],
)
values_template=k8s_config(
name = "_values_replacement",
srcs = ["values.yaml"],
params = params,
containers = containers,
sub_charts = sub_charts,
deps = containers + sub_charts,
)
chart_template=k8s_config(
name = "_chart_replacement",
srcs = ["Chart.yaml"],
params = params,
containers = containers,
sub_charts = sub_charts,
deps = containers + sub_charts,
)
updated_files = genrule(
name = "_replaced_files",
deps = [values_template, chart_template],
srcs = {
'values': [values_template],
'chart': [chart_template],
},
outs = {
'values': ['values.yaml'],
'chart': ['Chart.yaml'],
},
cmd = f"mv $(location {values_template}) $OUTS_VALUES && mv $(location {chart_template}) $OUTS_CHART",
)

pre_dep_chart_files=filegroup(
name = f'_pre_deps_files',
deps = [chart_tmp_files, updated_files],
srcs = [
chart_tmp_files,
":_replaced_files|values",
":_replaced_files|chart"
Comment on lines +56 to +57
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: if you're passing both the named outputs, you can just use :_replaced_files

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I can tweak/test this. I was learning plz when doing this...

],
)


if not chart_name:
chart_name = f'$(basename $(out_dir {pre_dep_chart_files}))'
remote = f'{repo}'
chart_folder = f'./$(out_dir {pre_dep_chart_files})'
package_name = package_name()


if remote.startswith('file://'):
remote = f'{remote}/{chart_name}'

fqn = build_rule(
name = f'{name}_fqn',
pass_env = params.keys(),
cmd = f"""
REPOSITORY={remote}
if [[ $REPOSITORY = file* ]]; then
REPOSITORY="$REPOSITORY/{chart_name}"
fi
cat >> $OUT <<EOL
- name: "{chart_name}"\\n version: "{version}"\\n repository: "$REPOSITORY"
EOL
""",
outs = [f'{name}_fqn'],
labels = labels + ["helm-fqn"],
stamp = True,
visibility = visibility,
)




sub_charts_updated_deps = [f'{sc}_updated_deps' for sc in sub_charts]



update_rule = genrule(
name = f'{name}_updated_deps',
deps = [pre_dep_chart_files] + sub_charts + sub_charts_updated_deps,
srcs = {
'chart_files': [pre_dep_chart_files],
'subchart_files': sub_charts + sub_charts_updated_deps,
},
outs = ['charts', 'Chart.lock'],
pass_env = ["APP_VERSION", "CHART_OCI_REGISTRY", "CHART_OCI_REGISTRY_USERNAME", "CHART_OCI_REGISTRY_PASSWORD"],
cmd = f"""
set -e

export HELM_EXPERIMENTAL_OCI=1

echo {oci_registry_password} | helm registry login {oci_registry} --username {oci_registry_username} --password-stdin
helm dependency update {package_name}
Comment on lines +111 to +112
Copy link
Member

@Tatskaari Tatskaari Jan 4, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

From your comment here:

This requires env vars as passthrough to login to helm because
otherwise, the genrule cannot see the context from a previous login.

I'm just wondering what this context is. Why does it need to log in to access it?

I'm also wondering if we can get away with only doing the update when we run one of the script targets below?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Helm allows you define sub charts, which is roughly akin to dependencies in a language framework package manager. These do not point to a central repo automatically. You can use a local relative file path, e.g. file://../ which points to other charts in the same parent folder. Or you can point to an OCI compatible registry like ACR, ECR, GCR, etc.

Running the update command creates/updates a lock file. For starters, this part of the PR does smell a bit. I tried a few other variations of this and landed here. The ideal use case (I think) would be for the user to maintain this lock file in source they way one might maintain package-lock.json, etc. I believe helm supports semantic ranges, etc and has a similar process for bumping dependencies within a rule/pattern. I used the pleasings/docker build_def as a guide when writing this and as a result, the Chart.yaml file includes variable markers, e.g. ${VERSION} and targets e.g. //charts/subchart:mychart. This is similar to how the docker def implements a "base image". When the source file is doctored with plz syntax like this it breaks the file. docker build with a Dockerfile that uses a plz target as a base image will break unless it is run using the special plz run target. In the same way, if I run helm update with these changes, it breaks.

Hopefully this explains more why it is this way. This led me to dynamically build this lock file inside the please context each time. Happy to help talk through a better approach.

Copy link
Member

@Tatskaari Tatskaari Jan 10, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would we be able to avoid logging in if we could download or build all the charts we depend on in other build rules and then use local file:// based URLs instead of remote registries?

If so, we could label sub-charts with something like helm-chart:src/my_service/k8s/my_chart.tar.gz, and pick these up in a pre_build function. The pre-build function can then update your command to set up these file paths. We do something similar for linker flags:

cc_library(
    ...
    linker_flags = "--pthread",
)

Adds cc:ld:--pthread as a label to the target. The cc_binary() rule then picks these up in a pre-build via get_labels(name, "cc:"), and updates it's command in accordingly. It's a little grungy but works quite well.

https://github.com/thought-machine/please/blob/master/rules/cc_rules.build_defs#L742

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think that makes sense. The other rule would still need to login though if I understand correctly. For our use case it doesn't make sense to switch to file:// since we want to publish the new chart with the correct remote dependency.

Copy link
Member

@Tatskaari Tatskaari Jan 10, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If this works, I'm reasonably happy to merge this and iterate as we go forward. I'm planning on moving rules out of the pleasings repo into their own repo (using the new plugin API) as they become mature enough. Ideally we'd not need to set up env variables just to build this stuff by then.

mkdir -p {package_name}/charts
mkdir -p build
cp -r {package_name}/charts/. build/
cp {package_name}/Chart.lock ./
rm -rf charts
mkdir -p charts
cp -r build/. charts/
""",
)

chart_files=filegroup(
name = f'{name}',
deps = [pre_dep_chart_files, update_rule],
labels = ["helm-chart"],
srcs = [
pre_dep_chart_files,
update_rule
],
)

package_rule = genrule(
name = "_helm_package",
deps = [chart_files],
srcs = {
'chart_files': [chart_files],
'subchart_files': sub_charts,
},
output_dirs = ["_out"],
pass_env = ["APP_VERSION"],
cmd = f"""
helm package {package_name} -d _out
""",
)

sub_charts_push = [f'{sc}_helm_push' for sc in sub_charts]
sub_chart_push_cmds='$(out_location %s)' % ';\n'.join(sub_charts_push)
sh_cmd(
name = f'{name}_helm_push',
labels = ["helm-push"],
deps = [pre_dep_chart_files] + sub_charts_push,
srcs = [pre_dep_chart_files],
cmd = f"""
export HELM_EXPERIMENTAL_OCI=1
{sub_chart_push_cmds}
helm dependency update {chart_folder}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Given that we're doing helm dependency update here, do we need to run the update at build time? It would simplify things if we don't.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think the reason I ended up like this (running it in two places) is because for awhile I was attempting a process that let me run plz to maintain the lock file in source. I think you're right though. I should simplify it. Assuming the design doesn't change around managing the lock file.

helm package {chart_folder} -d {chart_folder}
helm push {chart_folder}/{chart_name}-{version}.tgz {remote}
""",
)

sub_charts_update = [f'{sc}_helm_dependency_update' for sc in sub_charts]
sub_chart_update_cmds='$(out_location %s)' % ';\n'.join(sub_charts_update)
sh_cmd(
name = f'{name}_helm_dependency_update',
labels = ["helm-dependency-update"],
deps = [pre_dep_chart_files] + sub_charts_update,
srcs = [pre_dep_chart_files],
cmd = f"""
export HELM_EXPERIMENTAL_OCI=1
{sub_chart_update_cmds}
helm dependency update {chart_folder}
""",
)
sh_cmd(
name = "_helm_chart_debug",
labels = ["helm-chart-debug"],
deps = [package_rule, chart_files] + sub_charts + sub_charts_push,
srcs = {
'chart_files': [chart_files],
'subchart_files': sub_charts,
},
cmd = f"""
echo $(out_locations {package_rule})


""",
)

#

return chart_files

def deployment(scenario:str, chart:str, host_suffix:str='', namespace:str, deps:list=[]):
if chart:
chart = canonicalise(chart)

chart_folder = f'./$(out_dir {chart})'

if host_suffix:
release_name = f'{scenario}'
else:
release_name = f'{scenario}-{host_suffix}'

sh_cmd(
name = f'{scenario}_upgrade',
labels = ["helm-upgrade"],
deps = [chart],
srcs = {
'chart_files': [chart],
'scenario': [f'{scenario}.yaml']
},
cmd = f"""
helm upgrade {release_name} {chart_folder} -f $SRCS_SCENARIO --set global.hosts.hostSuffix={host_suffix} -n {namespace}
""",
)

sh_cmd(
name = f'{scenario}_install',
labels = ["helm-install"],
deps = [chart],
srcs = {
'chart_files': [chart],
'scenario': [f'{scenario}.yaml']
},
cmd = f"""
helm install -f $SRCS_SCENARIO {release_name} {chart_folder} --set global.hosts.hostSuffix={host_suffix} -n {namespace}
""",
)

sh_cmd(
name = f'{scenario}_uninstall',
labels = ["helm-uninstall"],
cmd = f"""
helm uninstall {release_name} -n {namespace}
""",
)
5 changes: 5 additions & 0 deletions helm/k8s_fork/BUILD
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
filegroup(
name = "k8s_fork",
srcs = ["k8s.build_defs"],
visibility = ["PUBLIC"],
)
Loading