Merge branch 'main' into report_improvements

.github/workflows/rill-cloud.yml

@@ -13,6 +13,7 @@ on:
 
 env:
   RELEASE: ${{ startsWith(github.ref, 'refs/tags/v') }}
+  DEPLOY_CLOUD: 1
 
 jobs:
   release:
@@ -39,10 +40,15 @@ jobs:
 
       - name: Build & Publish Rill docker image
         run: |-
+          if [ "$RELEASE" == "true" ] || [ "$GITHUB_REF_NAME" == "main" ]; then
+            echo "DEPLOY_CLOUD=1" >> $GITHUB_ENV
+          else
+            echo "DEPLOY_CLOUD=$(git branch -r | grep release-0 | sort | tail -1 | grep -c $GITHUB_REF_NAME)" >> $GITHUB_ENV
+          fi
+          
           if [ ${RELEASE} == "false" ]; then
             echo "Fetch tags to get the last tagged version"
-            git fetch --all --tags;            
-            echo "LATEST_BRANCH=$(git branch -r | grep release-0 | sort | tail -1 | grep -c ${GITHUB_REF_NAME})" >> $GITHUB_ENV
+            git fetch --all --tags;
           fi
 
           # Embed DuckDB extensions in the Rill binary
@@ -64,7 +70,7 @@ jobs:
           fi
 
       - name: Trigger Rill Cloud deployment
-        if: env.LATEST_BRANCH == '1'
+        if: env.DEPLOY_CLOUD == '1'
         run: |-
           set -e
           curl -X POST https://api.github.com/repos/rilldata/rill-helm-charts/dispatches \

admin/billing.go

@@ -34,6 +34,7 @@ func (s *Service) InitOrganizationBilling(ctx context.Context, org *database.Org
 		Name:                                org.Name,
 		DisplayName:                         org.DisplayName,
 		Description:                         org.Description,
+		LogoAssetID:                         org.LogoAssetID,
 		CustomDomain:                        org.CustomDomain,
 		QuotaProjects:                       org.QuotaProjects,
 		QuotaDeployments:                    org.QuotaDeployments,
@@ -124,6 +125,7 @@ func (s *Service) RepairOrganizationBilling(ctx context.Context, org *database.O
 		Name:                                org.Name,
 		DisplayName:                         org.DisplayName,
 		Description:                         org.Description,
+		LogoAssetID:                         org.LogoAssetID,
 		CustomDomain:                        org.CustomDomain,
 		QuotaProjects:                       org.QuotaProjects,
 		QuotaDeployments:                    org.QuotaDeployments,
@@ -181,6 +183,7 @@ func (s *Service) RepairOrganizationBilling(ctx context.Context, org *database.O
 			Name:                                org.Name,
 			DisplayName:                         org.DisplayName,
 			Description:                         org.Description,
+			LogoAssetID:                         org.LogoAssetID,
 			CustomDomain:                        org.CustomDomain,
 			QuotaProjects:                       biggerOfInt(sub.Plan.Quotas.NumProjects, org.QuotaProjects),
 			QuotaDeployments:                    biggerOfInt(sub.Plan.Quotas.NumDeployments, org.QuotaDeployments),
@@ -247,6 +250,7 @@ func (s *Service) StartTrial(ctx context.Context, org *database.Organization) (*
 		Name:                                org.Name,
 		DisplayName:                         org.DisplayName,
 		Description:                         org.Description,
+		LogoAssetID:                         org.LogoAssetID,
 		CustomDomain:                        org.CustomDomain,
 		QuotaProjects:                       biggerOfInt(plan.Quotas.NumProjects, org.QuotaProjects),
 		QuotaDeployments:                    biggerOfInt(plan.Quotas.NumDeployments, org.QuotaDeployments),

admin/database/database.go

@@ -265,7 +265,7 @@ type DB interface {
 
 	FindAsset(ctx context.Context, id string) (*Asset, error)
 	FindUnusedAssets(ctx context.Context, limit int) ([]*Asset, error)
-	InsertAsset(ctx context.Context, organizationID, path, ownerID string) (*Asset, error)
+	InsertAsset(ctx context.Context, id string, organizationID, path, ownerID string, cacheable bool) (*Asset, error)
 	DeleteAssets(ctx context.Context, ids []string) error
 
 	FindOrganizationIDsWithBilling(ctx context.Context) ([]string, error)
@@ -293,6 +293,7 @@ type DB interface {
 	DeleteProjectVariables(ctx context.Context, projectID, environment string, vars []string) error
 
 	FindProvisionerResourcesForDeployment(ctx context.Context, deploymentID string) ([]*ProvisionerResource, error)
+	FindProvisionerResourceByTypeAndName(ctx context.Context, deploymentID, typ, name string) (*ProvisionerResource, error)
 	InsertProvisionerResource(ctx context.Context, opts *InsertProvisionerResourceOptions) (*ProvisionerResource, error)
 	UpdateProvisionerResource(ctx context.Context, id string, opts *UpdateProvisionerResourceOptions) (*ProvisionerResource, error)
 	DeleteProvisionerResource(ctx context.Context, id string) error
@@ -315,6 +316,7 @@ type Organization struct {
 	Name                                string
 	DisplayName                         string `db:"display_name"`
 	Description                         string
+	LogoAssetID                         *string   `db:"logo_asset_id"`
 	CustomDomain                        string    `db:"custom_domain"`
 	AllUsergroupID                      *string   `db:"all_usergroup_id"`
 	CreatedOn                           time.Time `db:"created_on"`
@@ -336,6 +338,7 @@ type InsertOrganizationOptions struct {
 	Name                                string `validate:"slug"`
 	DisplayName                         string
 	Description                         string
+	LogoAssetID                         *string
 	CustomDomain                        string `validate:"omitempty,fqdn"`
 	QuotaProjects                       int
 	QuotaDeployments                    int
@@ -354,6 +357,7 @@ type UpdateOrganizationOptions struct {
 	Name                                string `validate:"slug"`
 	DisplayName                         string
 	Description                         string
+	LogoAssetID                         *string
 	CustomDomain                        string `validate:"omitempty,fqdn"`
 	QuotaProjects                       int
 	QuotaDeployments                    int
@@ -786,26 +790,28 @@ type OrganizationRole struct {
 
 // ProjectRole represents roles for projects.
 type ProjectRole struct {
-	ID                    string
-	Name                  string
-	ReadProject           bool `db:"read_project"`
-	ManageProject         bool `db:"manage_project"`
-	ReadProd              bool `db:"read_prod"`
-	ReadProdStatus        bool `db:"read_prod_status"`
-	ManageProd            bool `db:"manage_prod"`
-	ReadDev               bool `db:"read_dev"`
-	ReadDevStatus         bool `db:"read_dev_status"`
-	ManageDev             bool `db:"manage_dev"`
-	ReadProjectMembers    bool `db:"read_project_members"`
-	ManageProjectMembers  bool `db:"manage_project_members"`
-	CreateMagicAuthTokens bool `db:"create_magic_auth_tokens"`
-	ManageMagicAuthTokens bool `db:"manage_magic_auth_tokens"`
-	CreateReports         bool `db:"create_reports"`
-	ManageReports         bool `db:"manage_reports"`
-	CreateAlerts          bool `db:"create_alerts"`
-	ManageAlerts          bool `db:"manage_alerts"`
-	CreateBookmarks       bool `db:"create_bookmarks"`
-	ManageBookmarks       bool `db:"manage_bookmarks"`
+	ID                         string
+	Name                       string
+	ReadProject                bool `db:"read_project"`
+	ManageProject              bool `db:"manage_project"`
+	ReadProd                   bool `db:"read_prod"`
+	ReadProdStatus             bool `db:"read_prod_status"`
+	ManageProd                 bool `db:"manage_prod"`
+	ReadDev                    bool `db:"read_dev"`
+	ReadDevStatus              bool `db:"read_dev_status"`
+	ManageDev                  bool `db:"manage_dev"`
+	ReadProvisionerResources   bool `db:"read_provisioner_resources"`
+	ManageProvisionerResources bool `db:"manage_provisioner_resources"`
+	ReadProjectMembers         bool `db:"read_project_members"`
+	ManageProjectMembers       bool `db:"manage_project_members"`
+	CreateMagicAuthTokens      bool `db:"create_magic_auth_tokens"`
+	ManageMagicAuthTokens      bool `db:"manage_magic_auth_tokens"`
+	CreateReports              bool `db:"create_reports"`
+	ManageReports              bool `db:"manage_reports"`
+	CreateAlerts               bool `db:"create_alerts"`
+	ManageAlerts               bool `db:"manage_alerts"`
+	CreateBookmarks            bool `db:"create_bookmarks"`
+	ManageBookmarks            bool `db:"manage_bookmarks"`
 }
 
 // MemberUser is a convenience type used for display-friendly representation of an org or project member.
@@ -992,14 +998,18 @@ type InsertVirtualFileOptions struct {
 	Data      []byte `validate:"max=8192"` // 8kb
 }
 
+// Asset represents a user-uploaded file asset.
+// For example, this can be an upload deploy of a project or a custom logo for an org.
 type Asset struct {
 	ID             string
 	OrganizationID *string   `db:"org_id"`
 	Path           string    `db:"path"`
 	OwnerID        string    `db:"owner_id"`
+	Cacheable      bool      `db:"cacheable"`
 	CreatedOn      time.Time `db:"created_on"`
 }
 
+// ProjectVariable represents a key-value variable for a project, possible for a specific environment (e.g. production or development).
 type ProjectVariable struct {
 	ID                   string    `db:"id"`
 	ProjectID            string    `db:"project_id"`

admin/database/postgres/migrations/0055.sql

@@ -1,12 +1,5 @@
-ALTER TABLE magic_auth_tokens ADD COLUMN resources JSONB NOT NULL DEFAULT '[]'::jsonb;
-
-UPDATE magic_auth_tokens
-SET resources = jsonb_build_array(
-        jsonb_build_object(
-                'Type', resource_type,
-                'Name', resource_name
-        )
-);
-
-ALTER TABLE magic_auth_tokens DROP COLUMN resource_name, DROP COLUMN resource_type;
+ALTER TABLE project_roles ADD read_provisioner_resources BOOLEAN DEFAULT false NOT NULL;
+UPDATE project_roles SET read_provisioner_resources = read_prod_status;
 
+ALTER TABLE project_roles ADD manage_provisioner_resources BOOLEAN DEFAULT false NOT NULL;
+UPDATE project_roles SET manage_provisioner_resources = manage_prod;

admin/database/postgres/migrations/0056.sql

@@ -0,0 +1,3 @@
+ALTER TABLE orgs ADD COLUMN logo_asset_id UUID REFERENCES assets(id) ON DELETE SET NULL;
+
+ALTER TABLE assets ADD COLUMN cacheable BOOLEAN NOT NULL DEFAULT FALSE;

admin/database/postgres/migrations/0057.sql

@@ -0,0 +1,11 @@
+ALTER TABLE magic_auth_tokens ADD COLUMN resources JSONB NOT NULL DEFAULT '[]'::jsonb;
+
+UPDATE magic_auth_tokens
+SET resources = jsonb_build_array(
+        jsonb_build_object(
+                'Type', resource_type,
+                'Name', resource_name
+        )
+);
+
+ALTER TABLE magic_auth_tokens DROP COLUMN resource_name, DROP COLUMN resource_type;

admin/database/postgres/postgres.go

@@ -138,9 +138,9 @@ func (c *connection) InsertOrganization(ctx context.Context, opts *database.Inse
 	}
 
 	res := &database.Organization{}
-	err := c.getDB(ctx).QueryRowxContext(ctx, `INSERT INTO orgs(name, display_name, description, custom_domain, quota_projects, quota_deployments, quota_slots_total, quota_slots_per_deployment, quota_outstanding_invites, quota_storage_limit_bytes_per_deployment, billing_customer_id, payment_customer_id, billing_email, created_by_user_id)
-		VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) RETURNING *`,
-		opts.Name, opts.DisplayName, opts.Description, opts.CustomDomain, opts.QuotaProjects, opts.QuotaDeployments, opts.QuotaSlotsTotal, opts.QuotaSlotsPerDeployment, opts.QuotaOutstandingInvites, opts.QuotaStorageLimitBytesPerDeployment, opts.BillingCustomerID, opts.PaymentCustomerID, opts.BillingEmail, opts.CreatedByUserID).StructScan(res)
+	err := c.getDB(ctx).QueryRowxContext(ctx, `INSERT INTO orgs(name, display_name, description, logo_asset_id, custom_domain, quota_projects, quota_deployments, quota_slots_total, quota_slots_per_deployment, quota_outstanding_invites, quota_storage_limit_bytes_per_deployment, billing_customer_id, payment_customer_id, billing_email, created_by_user_id)
+		VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) RETURNING *`,
+		opts.Name, opts.DisplayName, opts.Description, opts.LogoAssetID, opts.CustomDomain, opts.QuotaProjects, opts.QuotaDeployments, opts.QuotaSlotsTotal, opts.QuotaSlotsPerDeployment, opts.QuotaOutstandingInvites, opts.QuotaStorageLimitBytesPerDeployment, opts.BillingCustomerID, opts.PaymentCustomerID, opts.BillingEmail, opts.CreatedByUserID).StructScan(res)
 	if err != nil {
 		return nil, parseErr("org", err)
 	}
@@ -158,7 +158,9 @@ func (c *connection) UpdateOrganization(ctx context.Context, id string, opts *da
 	}
 
 	res := &database.Organization{}
-	err := c.getDB(ctx).QueryRowxContext(ctx, "UPDATE orgs SET name=$1, display_name=$2, description=$3, custom_domain=$4, quota_projects=$5, quota_deployments=$6, quota_slots_total=$7, quota_slots_per_deployment=$8, quota_outstanding_invites=$9, quota_storage_limit_bytes_per_deployment=$10, billing_customer_id=$11, payment_customer_id=$12, billing_email=$13, created_by_user_id=$14, updated_on=now() WHERE id=$15 RETURNING *", opts.Name, opts.DisplayName, opts.Description, opts.CustomDomain, opts.QuotaProjects, opts.QuotaDeployments, opts.QuotaSlotsTotal, opts.QuotaSlotsPerDeployment, opts.QuotaOutstandingInvites, opts.QuotaStorageLimitBytesPerDeployment, opts.BillingCustomerID, opts.PaymentCustomerID, opts.BillingEmail, opts.CreatedByUserID, id).StructScan(res)
+	err := c.getDB(ctx).QueryRowxContext(ctx,
+		`UPDATE orgs SET name=$1, display_name=$2, description=$3, logo_asset_id=$4, custom_domain=$5, quota_projects=$6, quota_deployments=$7, quota_slots_total=$8, quota_slots_per_deployment=$9, quota_outstanding_invites=$10, quota_storage_limit_bytes_per_deployment=$11, billing_customer_id=$12, payment_customer_id=$13, billing_email=$14, created_by_user_id=$15, updated_on=now() WHERE id=$16 RETURNING *`,
+		opts.Name, opts.DisplayName, opts.Description, opts.LogoAssetID, opts.CustomDomain, opts.QuotaProjects, opts.QuotaDeployments, opts.QuotaSlotsTotal, opts.QuotaSlotsPerDeployment, opts.QuotaOutstandingInvites, opts.QuotaStorageLimitBytesPerDeployment, opts.BillingCustomerID, opts.PaymentCustomerID, opts.BillingEmail, opts.CreatedByUserID, id).StructScan(res)
 	if err != nil {
 		return nil, parseErr("org", err)
 	}
@@ -1936,12 +1938,12 @@ func (c *connection) FindAsset(ctx context.Context, id string) (*database.Asset,
 	return res, nil
 }
 
-func (c *connection) InsertAsset(ctx context.Context, organizationID, path, ownerID string) (*database.Asset, error) {
+func (c *connection) InsertAsset(ctx context.Context, id, organizationID, path, ownerID string, cacheable bool) (*database.Asset, error) {
 	res := &database.Asset{}
 	err := c.getDB(ctx).QueryRowxContext(ctx, `
-		INSERT INTO assets (org_id, path, owner_id)
-		VALUES ($1, $2, $3) RETURNING *`,
-		organizationID, path, ownerID,
+		INSERT INTO assets (id, org_id, path, owner_id, cacheable)
+		VALUES ($1, $2, $3, $4, $5) RETURNING *`,
+		id, organizationID, path, ownerID, cacheable,
 	).StructScan(res)
 	if err != nil {
 		return nil, parseErr("asset", err)
@@ -1956,8 +1958,8 @@ func (c *connection) FindUnusedAssets(ctx context.Context, limit int) ([]*databa
 	err := c.getDB(ctx).SelectContext(ctx, &res, `
 		SELECT a.* FROM assets a 
 		WHERE a.created_on < now() - INTERVAL '6 hours'
-		AND NOT EXISTS 
-		(SELECT 1 FROM projects p WHERE p.archive_asset_id = a.id)
+		AND NOT EXISTS (SELECT 1 FROM projects p WHERE p.archive_asset_id = a.id)
+		AND NOT EXISTS (SELECT 1 FROM orgs o WHERE o.logo_asset_id = a.id)
 		ORDER BY a.created_on DESC LIMIT $1
 	`, limit)
 	if err != nil {
@@ -2237,6 +2239,15 @@ func (c *connection) FindProvisionerResourcesForDeployment(ctx context.Context,
 	return c.provisionerResourcesFromDTOs(res)
 }
 
+func (c *connection) FindProvisionerResourceByTypeAndName(ctx context.Context, deploymentID, typ, name string) (*database.ProvisionerResource, error) {
+	res := &provisionerResourceDTO{}
+	err := c.getDB(ctx).QueryRowxContext(ctx, `SELECT * FROM provisioner_resources WHERE deployment_id = $1 AND "type" = $2 AND name = $3`, deploymentID, typ, name).StructScan(res)
+	if err != nil {
+		return nil, parseErr("provisioner resource", err)
+	}
+	return c.provisionerResourceFromDTO(res)
+}
+
 func (c *connection) InsertProvisionerResource(ctx context.Context, opts *database.InsertProvisionerResourceOptions) (*database.ProvisionerResource, error) {
 	if err := database.Validate(opts); err != nil {
 		return nil, err