diff --git a/d2graph/d2graph.go b/d2graph/d2graph.go
index b954199fc0..dce1e06243 100644
--- a/d2graph/d2graph.go
+++ b/d2graph/d2graph.go
@@ -1471,10 +1471,16 @@ func (g *Graph) SetDimensions(mtexts []*d2target.MText, ruler *textmeasure.Ruler
 		}
 	}
 	for _, edge := range g.Edges {
+		usedFont := fontFamily
+		if edge.Style.Font != nil {
+			f := d2fonts.D2_FONT_TO_FAMILY[edge.Style.Font.Value]
+			usedFont = &f
+		}
+
 		if edge.SrcArrowhead != nil && edge.SrcArrowhead.Label.Value != "" {
 			t := edge.Text()
 			t.Text = edge.SrcArrowhead.Label.Value
-			dims := GetTextDimensions(mtexts, ruler, t, fontFamily)
+			dims := GetTextDimensions(mtexts, ruler, t, usedFont)
 			edge.MinWidth += dims.Width + INNER_LABEL_PADDING
 			edge.MinHeight += dims.Height + INNER_LABEL_PADDING
 			edge.SrcArrowhead.LabelDimensions = *dims
@@ -1482,7 +1488,7 @@ func (g *Graph) SetDimensions(mtexts []*d2target.MText, ruler *textmeasure.Ruler
 		if edge.DstArrowhead != nil && edge.DstArrowhead.Label.Value != "" {
 			t := edge.Text()
 			t.Text = edge.DstArrowhead.Label.Value
-			dims := GetTextDimensions(mtexts, ruler, t, fontFamily)
+			dims := GetTextDimensions(mtexts, ruler, t, usedFont)
 			edge.MinWidth += dims.Width + INNER_LABEL_PADDING
 			edge.MinHeight += dims.Height + INNER_LABEL_PADDING
 			edge.DstArrowhead.LabelDimensions = *dims
@@ -1497,17 +1503,12 @@ func (g *Graph) SetDimensions(mtexts []*d2target.MText, ruler *textmeasure.Ruler
 		}
 		edge.ApplyTextTransform()
 
-		usedFont := fontFamily
-		if edge.Style.Font != nil {
-			f := d2fonts.D2_FONT_TO_FAMILY[edge.Style.Font.Value]
-			usedFont = &f
-		}
-
 		dims := GetTextDimensions(mtexts, ruler, edge.Text(), usedFont)
 		if dims == nil {
 			return fmt.Errorf("dimensions for edge label %#v not found", edge.Text())
 		}
 
+		fmt.Printf("edge dims %v\n", *dims)
 		edge.LabelDimensions = *dims
 		edge.MinWidth += dims.Width
 		edge.MinHeight += dims.Height
diff --git a/d2renderers/d2svg/d2svg.go b/d2renderers/d2svg/d2svg.go
index 3e38c8cf36..2252194484 100644
--- a/d2renderers/d2svg/d2svg.go
+++ b/d2renderers/d2svg/d2svg.go
@@ -620,42 +620,99 @@ func drawConnection(writer io.Writer, labelMaskID string, connection d2target.Co
 		fmt.Fprint(writer, textEl.Render())
 	}
 
-	length := geo.Route(connection.Route).Length()
 	if connection.SrcLabel != nil && connection.SrcLabel.Label != "" {
 		// TODO use arrowhead label dimensions https://github.com/terrastruct/d2/issues/183
-		// size := float64(connection.FontSize)
-		width := float64(connection.SrcLabel.LabelWidth)
-		height := float64(connection.DstLabel.LabelHeight)
-		position := 0.
-		if length > 0 {
-			position = math.Max(width, height) / length
-		}
-		fmt.Fprint(writer, renderArrowheadLabel(connection, connection.SrcLabel.Label, position, width, height))
+		fmt.Fprint(writer, renderArrowheadLabel(connection, connection.SrcLabel.Label, false))
 	}
 	if connection.DstLabel != nil && connection.DstLabel.Label != "" {
 		// TODO use arrowhead label dimensions https://github.com/terrastruct/d2/issues/183
-		// size := float64(connection.FontSize)
-		width := float64(connection.DstLabel.LabelWidth)
-		height := float64(connection.DstLabel.LabelHeight)
-		position := 1.
-		if length > 0 {
-			position -= math.Max(width, height) / length
-		}
-		fmt.Fprint(writer, renderArrowheadLabel(connection, connection.DstLabel.Label, position, width, height))
+		fmt.Fprint(writer, renderArrowheadLabel(connection, connection.DstLabel.Label, true))
 	}
 	fmt.Fprintf(writer, `</g>`)
 	return
 }
 
-func renderArrowheadLabel(connection d2target.Connection, text string, position, width, height float64) string {
-	labelTL := label.UnlockedTop.GetPointOnRoute(connection.Route, float64(connection.StrokeWidth), position, width, height)
+func renderArrowheadLabel(connection d2target.Connection, text string, isDst bool) string {
+	var width, height float64
+	if isDst {
+		width = float64(connection.DstLabel.LabelWidth)
+		height = float64(connection.DstLabel.LabelHeight)
+	} else {
+		width = float64(connection.SrcLabel.LabelWidth)
+		height = float64(connection.SrcLabel.LabelHeight)
+	}
+
+	// get the start/end points of edge segment with arrowhead
+	index := 0
+	if isDst {
+		index = len(connection.Route) - 2
+	}
+	start, end := connection.Route[index], connection.Route[index+1]
+
+	// how much to move the label back from the very end of the edge
+	var shift float64
+	if start.Y == end.Y {
+		// shift left/right to fit on horizontal segment
+		shift = width/2. + label.PADDING
+	} else if start.X == end.X {
+		// shift up/down to fit on vertical segment
+		shift = height/2. + label.PADDING
+	} else {
+		// TODO compute amount to shift according to angle instead of max
+		shift = math.Max(width, height)
+	}
+
+	length := geo.Route(connection.Route).Length()
+	var position float64
+	if isDst {
+		position = 1.
+		if length > 0 {
+			position -= shift / length
+		}
+	} else {
+		position = 0.
+		if length > 0 {
+			position = shift / length
+		}
+	}
+
+	labelTL, index := label.UnlockedTop.GetPointOnRoute(connection.Route, float64(connection.StrokeWidth), position, width, height)
+
+	// svg text is positioned with the center of its baseline
+	baselineCenter := geo.Point{
+		X: labelTL.X + width/2.,
+		Y: labelTL.Y + float64(connection.FontSize),
+	}
+
+	var arrowheadOffset float64
+	if isDst && connection.DstArrow != d2target.NoArrowhead {
+		// TODO offset according to arrowhead dimensions
+		arrowheadOffset = 5
+	} else if connection.SrcArrow != d2target.NoArrowhead {
+		arrowheadOffset = 5
+	}
+
+	var offsetX, offsetY float64
+	if start.Y == end.Y {
+		// shift up/down over horizontal segment
+		offsetY = arrowheadOffset
+		if end.Y < start.Y {
+			offsetY = -offsetY
+		}
+	} else if start.X == end.X {
+		// shift left/right across vertical segment
+		offsetX = arrowheadOffset
+		if end.X < start.X {
+			offsetX = -offsetX
+		}
+	}
 
 	textEl := d2themes.NewThemableElement("text")
-	textEl.X = labelTL.X + width/2
-	textEl.Y = labelTL.Y + float64(connection.FontSize)
+	textEl.X = baselineCenter.X + offsetX
+	textEl.Y = baselineCenter.Y + offsetY
 	textEl.Fill = d2target.FG_COLOR
 	textEl.ClassName = "text-italic"
-	textEl.Style = fmt.Sprintf("text-anchor:%s;font-size:%vpx", "middle", connection.FontSize)
+	textEl.Style = fmt.Sprintf("text-anchor:middle;font-size:%vpx", connection.FontSize)
 	textEl.Content = RenderText(text, textEl.X, height)
 	return textEl.Render()
 }
diff --git a/d2target/d2target.go b/d2target/d2target.go
index bd5c494112..b1ed4a4af5 100644
--- a/d2target/d2target.go
+++ b/d2target/d2target.go
@@ -510,13 +510,14 @@ func (c Connection) CSSStyle() string {
 }
 
 func (c *Connection) GetLabelTopLeft() *geo.Point {
-	return label.Position(c.LabelPosition).GetPointOnRoute(
+	point, _ := label.Position(c.LabelPosition).GetPointOnRoute(
 		c.Route,
 		float64(c.StrokeWidth),
 		c.LabelPercentage,
 		float64(c.LabelWidth),
 		float64(c.LabelHeight),
 	)
+	return point
 }
 
 func (c Connection) GetZIndex() int {
diff --git a/e2etests/e2e_test.go b/e2etests/e2e_test.go
index ba19b4866e..bbf57a8ebb 100644
--- a/e2etests/e2e_test.go
+++ b/e2etests/e2e_test.go
@@ -209,6 +209,7 @@ func run(t *testing.T, tc testCase) {
 		renderOpts := &d2svg.RenderOpts{
 			Pad:     0,
 			ThemeID: tc.themeID,
+			// SetDimensions: true,
 		}
 		if len(diagram.Layers) > 0 || len(diagram.Scenarios) > 0 || len(diagram.Steps) > 0 {
 			masterID, err := diagram.HashID()
diff --git a/lib/label/label.go b/lib/label/label.go
index fff93e1a0a..0dc134ac81 100644
--- a/lib/label/label.go
+++ b/lib/label/label.go
@@ -223,7 +223,7 @@ func (labelPosition Position) GetPointOnBox(box *geo.Box, padding, width, height
 }
 
 // return the top left point of a width x height label at the given label position on the route
-func (labelPosition Position) GetPointOnRoute(route geo.Route, strokeWidth, labelPercentage, width, height float64) *geo.Point {
+func (labelPosition Position) GetPointOnRoute(route geo.Route, strokeWidth, labelPercentage, width, height float64) (point *geo.Point, index int) {
 	totalLength := route.Length()
 	leftPosition := LEFT_LABEL_POSITION * totalLength
 	centerPosition := CENTER_LABEL_POSITION * totalLength
@@ -272,11 +272,11 @@ func (labelPosition Position) GetPointOnRoute(route geo.Route, strokeWidth, labe
 	var labelCenter *geo.Point
 	switch labelPosition {
 	case InsideMiddleLeft:
-		labelCenter, _ = route.GetPointAtDistance(leftPosition)
+		labelCenter, index = route.GetPointAtDistance(leftPosition)
 	case InsideMiddleCenter:
-		labelCenter, _ = route.GetPointAtDistance(centerPosition)
+		labelCenter, index = route.GetPointAtDistance(centerPosition)
 	case InsideMiddleRight:
-		labelCenter, _ = route.GetPointAtDistance(rightPosition)
+		labelCenter, index = route.GetPointAtDistance(rightPosition)
 
 	case OutsideTopLeft:
 		basePoint, index := route.GetPointAtDistance(leftPosition)
@@ -302,17 +302,17 @@ func (labelPosition Position) GetPointOnRoute(route geo.Route, strokeWidth, labe
 		basePoint, index := route.GetPointAtDistance(unlockedPosition)
 		labelCenter = getOffsetLabelPosition(basePoint, route[index], route[index+1], true)
 	case UnlockedMiddle:
-		labelCenter, _ = route.GetPointAtDistance(unlockedPosition)
+		labelCenter, index = route.GetPointAtDistance(unlockedPosition)
 	case UnlockedBottom:
 		basePoint, index := route.GetPointAtDistance(unlockedPosition)
 		labelCenter = getOffsetLabelPosition(basePoint, route[index], route[index+1], false)
 	default:
-		return nil
+		return nil, -1
 	}
 	// convert from center to top left
 	labelCenter.X = chopPrecision(labelCenter.X - width/2)
 	labelCenter.Y = chopPrecision(labelCenter.Y - height/2)
-	return labelCenter
+	return labelCenter, index
 }
 
 // TODO probably use math.Big