This stylesheet:
<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
<xsl:output method="xml" omit-xml-declaration="yes"/>
<xsl:template match="@*|node()">
<xsl:copy>
<xsl:apply-templates select="@*|node()"/>
</xsl:copy>
</xsl:template>
<xsl:template match="li">
<xsl:copy>
<xsl:apply-templates select="@*"/>
<xsl:variable name="class">
<xsl:if test="position() > last() - last() mod 3 - 3 * not(last() mod 3)">row-last </xsl:if>
<xsl:if test="not(position() mod 3)">col-last </xsl:if>
</xsl:variable>
<xsl:if test="$class != ''">
<xsl:attribute name="class">
<xsl:value-of select="normalize-space($class)"/>
</xsl:attribute>
</xsl:if>
<xsl:apply-templates select="node()"/>
</xsl:copy>
</xsl:template>
</xsl:stylesheet>
With these inputs:
<ul>
<li>Item 1</li>
<li>Item 2</li>
<li>Item 3</li>
<li>Item 4</li>
<li>Item 5</li>
<li>Item 6</li>
</ul>
<ul>
<li>Item 1</li>
<li>Item 2</li>
<li>Item 3</li>
<li>Item 4</li>
<li>Item 5</li>
</ul>
<ul>
<li>Item 1</li>
<li>Item 2</li>
<li>Item 3</li>
<li>Item 4</li>
</ul>
Results:
<ul>
<li>Item 1</li>
<li>Item 2</li>
<li class="col-last">Item 3</li>
<li class="row-last">Item 4</li>
<li class="row-last">Item 5</li>
<li class="row-last col-last">Item 6</li>
</ul>
<ul>
<li>Item 1</li>
<li>Item 2</li>
<li class="col-last">Item 3</li>
<li class="row-last">Item 4</li>
<li class="row-last">Item 5</li>
</ul>
<ul>
<li>Item 1</li>
<li>Item 2</li>
<li class="col-last">Item 3</li>
<li class="row-last">Item 4</li>
</ul>
Note: When ouput attributes take notice that last attribute with same name overwrites previous.