Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix backup functionality #2272

Draft
wants to merge 6 commits into
base: develop
Choose a base branch
from
Draft

Fix backup functionality #2272

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
a68e9e6
Remove non-functional backup instance for blobs
JimMadge Oct 30, 2024
d618f99
Remove incorrect docs section
JimMadge Nov 12, 2024
fd961a9
Merge remote-tracking branch 'origin/develop' into backup
JimMadge Nov 12, 2024
12dcc65
Remove old backup infrastructure
JimMadge Nov 12, 2024
ceb302a
WIP: Add backup VM
JimMadge Nov 12, 2024
0bead11
Merge remote-tracking branch 'origin/develop' into backup
JimMadge Dec 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
181 changes: 32 additions & 149 deletions data_safe_haven/infrastructure/programs/sre/backup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,10 @@

from collections.abc import Mapping

from pulumi import ComponentResource, Input, ResourceOptions
from pulumi_azure_native import dataprotection
from pulumi import ComponentResource, Input, Output, ResourceOptions

from data_safe_haven.infrastructure.components import LinuxVMComponentProps, VMComponent
from data_safe_haven.functions import b64encode, replace_separators


class SREBackupProps:
Expand Down Expand Up @@ -41,154 +43,35 @@ def __init__(
child_opts = ResourceOptions.merge(opts, ResourceOptions(parent=self))
child_tags = {"component": "backup"} | (tags if tags else {})

# Deploy backup vault
backup_vault = dataprotection.BackupVault(
f"{self._name}_backup_vault",
identity=dataprotection.DppIdentityDetailsArgs(
type="SystemAssigned",
),
location=props.location,
properties=dataprotection.BackupVaultArgs(
storage_settings=[
dataprotection.StorageSettingArgs(
datastore_type=dataprotection.StorageSettingStoreTypes.VAULT_STORE,
type=dataprotection.StorageSettingTypes.LOCALLY_REDUNDANT,
)
],
# Template cloud init
cloudinit = Output.all(
apt_proxy_server_hostname=props.apt_proxy_server_hostname,
storage_account_desired_state_name=props.storage_account_desired_state_name,
storage_account_data_private_user_name=props.storage_account_data_private_user_name,
storage_account_data_private_sensitive_name=props.storage_account_data_private_sensitive_name,
).apply(lambda kwargs: self.template_cloudinit(**kwargs))

# Backup virtual machine
VMComponent(
replace_separators(f"{self._name}_vm_backup", "_"),
LinuxVMComponentProps(
admin_password=props.admin_password,
admin_username=props.admin_username,
b64cloudinit=cloudinit.apply(b64encode),
data_collection_rule_id=props.data_collection_rule_id,
data_collection_endpoint_id=props.data_collection_endpoint_id,
ip_address_private=...,
location=props.location,
maintenance_configuration_id=props.maintenance_configuration_id,
resource_group_name=props.resource_group_name,
subnet_name=props.subnet_workspaces_name,
virtual_network_name=props.virtual_network_name,
virtual_network_resource_group_name=props.resource_group_name,
vm_name=Output.concat(
stack_name, "-vm-backup"
).apply(lambda s: replace_separators(s, "-")),
vm_size="Standard_B2s_v2",
),
resource_group_name=props.resource_group_name,
vault_name=f"{stack_name}-bv-backup",
opts=child_opts,
tags=child_tags,
)

# Backup policy for blobs
backup_policy_blobs = dataprotection.BackupPolicy(
f"{self._name}_backup_policy_blobs",
backup_policy_name="backup-policy-blobs",
properties=dataprotection.BackupPolicyArgs(
datasource_types=["Microsoft.Storage/storageAccounts/blobServices"],
object_type="BackupPolicy",
policy_rules=[
# Retain for 30 days
dataprotection.AzureRetentionRuleArgs(
is_default=True,
lifecycles=[
dataprotection.SourceLifeCycleArgs(
delete_after=dataprotection.AbsoluteDeleteOptionArgs(
duration="P30D",
object_type="AbsoluteDeleteOption",
),
source_data_store=dataprotection.DataStoreInfoBaseArgs(
data_store_type=dataprotection.DataStoreTypes.OPERATIONAL_STORE,
object_type="DataStoreInfoBase",
),
),
],
name="Default",
object_type="AzureRetentionRule",
),
],
),
resource_group_name=props.resource_group_name,
vault_name=backup_vault.name,
opts=ResourceOptions.merge(
child_opts, ResourceOptions(parent=backup_vault)
),
)

# Backup policy for disks
dataprotection.BackupPolicy(
f"{self._name}_backup_policy_disks",
backup_policy_name="backup-policy-disks",
properties=dataprotection.BackupPolicyArgs(
datasource_types=["Microsoft.Compute/disks"],
object_type="BackupPolicy",
policy_rules=[
# Backup at 02:00 every day
dataprotection.AzureBackupRuleArgs(
backup_parameters=dataprotection.AzureBackupParamsArgs(
backup_type="Incremental",
object_type="AzureBackupParams",
),
data_store=dataprotection.DataStoreInfoBaseArgs(
data_store_type=dataprotection.DataStoreTypes.OPERATIONAL_STORE,
object_type="DataStoreInfoBase",
),
name="BackupDaily",
object_type="AzureBackupRule",
trigger=dataprotection.ScheduleBasedTriggerContextArgs(
object_type="ScheduleBasedTriggerContext",
schedule=dataprotection.BackupScheduleArgs(
repeating_time_intervals=[
"R/2023-01-01T02:00:00+00:00/P1D"
],
),
tagging_criteria=[
dataprotection.TaggingCriteriaArgs(
is_default=True,
tag_info=dataprotection.RetentionTagArgs(
tag_name="Default",
),
tagging_priority=99,
)
],
),
),
# Retain for 30 days
dataprotection.AzureRetentionRuleArgs(
is_default=True,
lifecycles=[
dataprotection.SourceLifeCycleArgs(
delete_after=dataprotection.AbsoluteDeleteOptionArgs(
duration="P30D",
object_type="AbsoluteDeleteOption",
),
source_data_store=dataprotection.DataStoreInfoBaseArgs(
data_store_type=dataprotection.DataStoreTypes.OPERATIONAL_STORE,
object_type="DataStoreInfoBase",
),
),
],
name="Default",
object_type="AzureRetentionRule",
),
],
),
resource_group_name=props.resource_group_name,
vault_name=backup_vault.name,
opts=ResourceOptions.merge(
child_opts, ResourceOptions(parent=backup_vault)
),
)

# Backup instance for blobs
dataprotection.BackupInstance(
f"{self._name}_backup_instance_blobs",
backup_instance_name="backup-instance-blobs",
properties=dataprotection.BackupInstanceArgs(
data_source_info=dataprotection.DatasourceArgs(
resource_id=props.storage_account_data_private_sensitive_id,
datasource_type="Microsoft.Storage/storageAccounts/blobServices",
object_type="Datasource",
resource_location=props.location,
resource_name=props.storage_account_data_private_sensitive_name,
resource_type="Microsoft.Storage/storageAccounts",
resource_uri=props.storage_account_data_private_sensitive_id,
),
object_type="BackupInstance",
policy_info=dataprotection.PolicyInfoArgs(
policy_id=backup_policy_blobs.id,
),
friendly_name="BlobBackupSensitiveData",
),
resource_group_name=props.resource_group_name,
vault_name=backup_vault.name,
opts=ResourceOptions.merge(
child_opts, ResourceOptions(parent=backup_policy_blobs)
),
)

# Backup instance for disks
# We currently have no disks except OS disks so no backup is needed
# This may change in future, so we leave the policy above
17 changes: 0 additions & 17 deletions docs/source/deployment/deploy_sre.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -260,20 +260,3 @@ If you want to make changes to the config, edit this file and then run `dsh conf
:::{code} shell
$ dsh sre deploy YOUR_SRE_NAME
:::

::::{important}
After deployment, you may need to manually ensure that backups function.

- In the Azure Portal, navigate to the resource group for the SRE: **shm-_SHM\_NAME_-sre-_SRE\_NAME_-rg**
- Navigate to the backup vault for the SRE: **shm-_SHM\_NAME_-sre-_SRE\_NAME_-bv-backup**
- From the side menu, select **{menuselection}`Manage --> Backup Instances`**
- Change **Datasource type** to **Azure Blobs (Azure Storage)**
- Select the **BlobBackupSensitiveData** instance

If you see the message **Fix protection error for the backup instance**, as pictured below, then click the **{guilabel}`Fix protection error`** button.

:::{image} images/backup_fix_protection_error.png
:alt: Fix protection error for the backup instance
:align: center
:::
::::
Loading