Unfortunately, you can't use SystemColors.ControlTextBrushKey
because it applies when the item is unselected, or when it is selected but inactive (your question reads as though you're only interested in the latter). However, you can do this:
<ListBox ...>
<ListBox.Resources>
<!-- this customizes the background color when the item is selected but inactive -->
<SolidColorBrush x:Key="{x:Static SystemColors.ControlBrushKey}">Red</SolidColorBrush>
</ListBox.Resources>
<ListBox.ItemContainerStyle>
<Style>
<Style.Triggers>
<!-- this customizes the foreground color when the item is selected but inactive -->
<Trigger Property="Selector.IsSelected" Value="True">
<Setter Property="TextElement.Foreground" Value="Blue"/>
</Trigger>
</Style.Triggers>
</Style>
</ListBox.ItemContainerStyle>
</ListBox>
HTH,
Kent