By default tab items hide their right border when selected. This creates a visual appearance of it blending into the tab content. However, since you rotated the tab item 90 degrees it no longer serves this purpose. To get the correct visual reaction you will need to edit the TabItem's ControlTemplate. The easiest way to do this is to start with a copy of the template (if you have Blend you can simply go to Edit Template Copy). If you don't have Blend then you can use the default resources created by my version of Blend (below). Then edit the correct border properties to create the visual that you want.
<Window.Resources>
<Style x:Key="TabItemFocusVisual">
<Setter Property="Control.Template">
<Setter.Value>
<ControlTemplate>
<Rectangle SnapsToDevicePixels="true" Margin="3,3,3,1" Stroke="Black" StrokeDashArray="1 2" StrokeThickness="1"/>
</ControlTemplate>
</Setter.Value>
</Setter>
</Style>
<SolidColorBrush x:Key="TabControlNormalBorderBrush" Color="#8C8E94"/>
<LinearGradientBrush x:Key="ButtonNormalBackground" EndPoint="0,1" StartPoint="0,0">
<GradientStop Color="#F3F3F3" Offset="0"/>
<GradientStop Color="#EBEBEB" Offset="0.5"/>
<GradientStop Color="#DDDDDD" Offset="0.5"/>
<GradientStop Color="#CDCDCD" Offset="1"/>
</LinearGradientBrush>
<LinearGradientBrush x:Key="TabItemHotBackground" EndPoint="0,1" StartPoint="0,0">
<GradientStop Color="#EAF6FD" Offset="0.15"/>
<GradientStop Color="#D9F0FC" Offset=".5"/>
<GradientStop Color="#BEE6FD" Offset=".5"/>
<GradientStop Color="#A7D9F5" Offset="1"/>
</LinearGradientBrush>
<SolidColorBrush x:Key="TabItemSelectedBackground" Color="#F9F9F9"/>
<SolidColorBrush x:Key="TabItemHotBorderBrush" Color="#3C7FB1"/>
<SolidColorBrush x:Key="TabItemDisabledBackground" Color="#F4F4F4"/>
<SolidColorBrush x:Key="TabItemDisabledBorderBrush" Color="#FFC9C7BA"/>
<Style x:Key="TabItemStyle1" TargetType="{x:Type TabItem}">
<Setter Property="FocusVisualStyle" Value="{StaticResource TabItemFocusVisual}"/>
<Setter Property="Foreground" Value="Black"/>
<Setter Property="Padding" Value="6,1,6,1"/>
<Setter Property="BorderBrush" Value="{StaticResource TabControlNormalBorderBrush}"/>
<Setter Property="Background" Value="{StaticResource ButtonNormalBackground}"/>
<Setter Property="HorizontalContentAlignment" Value="Stretch"/>
<Setter Property="VerticalContentAlignment" Value="Stretch"/>
<Setter Property="Template">
<Setter.Value>
<ControlTemplate TargetType="{x:Type TabItem}">
<Grid SnapsToDevicePixels="true">
<Border x:Name="Bd" Background="{TemplateBinding Background}" BorderBrush="{TemplateBinding BorderBrush}" BorderThickness="1,1,1,0" Padding="{TemplateBinding Padding}">
<ContentPresenter SnapsToDevicePixels="{TemplateBinding SnapsToDevicePixels}" HorizontalAlignment="{Binding Path=HorizontalContentAlignment, RelativeSource={RelativeSource AncestorType={x:Type ItemsControl}}}" x:Name="Content" VerticalAlignment="{Binding Path=VerticalContentAlignment, RelativeSource={RelativeSource AncestorType={x:Type ItemsControl}}}" ContentSource="Header" RecognizesAccessKey="True"/>
</Border>
</Grid>
<ControlTemplate.Triggers>
<Trigger Property="IsMouseOver" Value="true">
<Setter Property="Background" TargetName="Bd" Value="{StaticResource TabItemHotBackground}"/>
</Trigger>
<Trigger Property="IsSelected" Value="true">
<Setter Property="Panel.ZIndex" Value="1"/>
<Setter Property="Background" TargetName="Bd" Value="{StaticResource TabItemSelectedBackground}"/>
</Trigger>
<MultiTrigger>
<MultiTrigger.Conditions>
<Condition Property="IsSelected" Value="false"/>
<Condition Property="IsMouseOver" Value="true"/>
</MultiTrigger.Conditions>
<Setter Property="BorderBrush" TargetName="Bd" Value="{StaticResource TabItemHotBorderBrush}"/>
</MultiTrigger>
<Trigger Property="TabStripPlacement" Value="Bottom">
<Setter Property="BorderThickness" TargetName="Bd" Value="1,0,1,1"/>
</Trigger>
<Trigger Property="TabStripPlacement" Value="Left">
<Setter Property="BorderThickness" TargetName="Bd" Value="1,1,0,1"/>
</Trigger>
<Trigger Property="TabStripPlacement" Value="Right">
<Setter Property="BorderThickness" TargetName="Bd" Value="0,1,1,1"/>
</Trigger>
<MultiTrigger>
<MultiTrigger.Conditions>
<Condition Property="IsSelected" Value="true"/>
<Condition Property="TabStripPlacement" Value="Top"/>
</MultiTrigger.Conditions>
<Setter Property="Margin" Value="-2,-2,-2,-1"/>
<Setter Property="Margin" TargetName="Content" Value="0,0,0,1"/>
</MultiTrigger>
<MultiTrigger>
<MultiTrigger.Conditions>
<Condition Property="IsSelected" Value="true"/>
<Condition Property="TabStripPlacement" Value="Bottom"/>
</MultiTrigger.Conditions>
<Setter Property="Margin" Value="-2,-1,-2,-2"/>
<Setter Property="Margin" TargetName="Content" Value="0,1,0,0"/>
</MultiTrigger>
<MultiTrigger>
<MultiTrigger.Conditions>
<Condition Property="IsSelected" Value="true"/>
<Condition Property="TabStripPlacement" Value="Left"/>
</MultiTrigger.Conditions>
<Setter Property="Margin" Value="-2,-2,-1,-2"/>
<Setter Property="Margin" TargetName="Content" Value="0,0,1,0"/>
</MultiTrigger>
<MultiTrigger>
<MultiTrigger.Conditions>
<Condition Property="IsSelected" Value="true"/>
<Condition Property="TabStripPlacement" Value="Right"/>
</MultiTrigger.Conditions>
<Setter Property="Margin" Value="-1,-2,-2,-2"/>
<Setter Property="Margin" TargetName="Content" Value="1,0,0,0"/>
</MultiTrigger>
<Trigger Property="IsEnabled" Value="false">
<Setter Property="Background" TargetName="Bd" Value="{StaticResource TabItemDisabledBackground}"/>
<Setter Property="BorderBrush" TargetName="Bd" Value="{StaticResource TabItemDisabledBorderBrush}"/>
<Setter Property="Foreground" Value="{DynamicResource {x:Static SystemColors.GrayTextBrushKey}}"/>
</Trigger>
</ControlTemplate.Triggers>
</ControlTemplate>
</Setter.Value>
</Setter>
</Style>
</Window.Resources>