The expression is not supported error when accessing Azure Tables

Question

I'm trying to get some records from the Azure Table Storage while using paging with the continuation token.

I have the following code:

public Stories SelectStory(DateTime start, DateTime end, string searchGuid)
{
  long startTicks = DateTime.MaxValue.Ticks - start.ToUniversalTime().Ticks;
  long endTicks = DateTime.MaxValue.Ticks - end.ToUniversalTime().Ticks;

  var stories = _ServiceContext.CreateQuery<Story>("Story").Where(s => Convert.ToInt64(s.RowKey.Substring(0, s.PartitionKey.IndexOf("_"))) > startTicks
         && Convert.ToInt64(s.RowKey.Substring(0, s.PartitionKey.IndexOf("_"))) < endTicks
         && s.RowKey == "story_" + searchGuid).Take(50);
  var query = stories as DataServiceQuery<Story>;
  var results = query.Execute();
  var response = results as QueryOperationResponse;

  Stories temp = new Stories();
  if(response.Headers.ContainsKey("x-ms-continuation-NextRowKey"))
  {
    temp.NextPartitionKey = response.Headers["x-ms-continuation-NextPartitionKey"];
    if (response.Headers.ContainsKey("x-ms-continuation-NextRowKey"))
    {
      temp.NextRowKey = response.Headers["x-ms-continuation-NextRowKey"];
    }
  }
  temp.List = results.ToList();

  return temp;
}

But I'm getting the following error:

 The expression (((ToInt64([10007].RowKey.Substring(0, [10007].PartitionKey.IndexOf("_"))) > 2521167043199999999) And (ToInt64([10007].RowKey.Substring(0, [10007].PartitionKey.IndexOf("_"))) < 2521154083199999999)) And ([10007].RowKey == "story_9")) is not supported. 

I'm not sure why the expression is not allowed. Does anyone have any ideas how I can change it to get it to work?

Thanks!

Edit: the new code (no errors but no data gets selected - even though i know it exists):

public Stories SelectStory(DateTime start, DateTime end, string searchGuid)
    {
        long startTicks = DateTime.MaxValue.Ticks - start.ToUniversalTime().Ticks;
        long endTicks = DateTime.MaxValue.Ticks - end.ToUniversalTime().Ticks;

        var strStart = string.Format("{0:10}_{1}", DateTime.MaxValue.Ticks - startTicks, "00000000-0000-0000-0000-000000000000");
        var strEnd = string.Format("{0:10}_{1}", DateTime.MaxValue.Ticks - endTicks, "00000000-0000-0000-0000-000000000000");

        var stories = _ServiceContext.CreateQuery<Story>("Story").Where(
                        s => s.RowKey.CompareTo(strStart) < 0
                            && s.RowKey.CompareTo(strEnd) > 0
                           //s.RowKey.CompareTo(startTicks.ToString() + "_") > 0
                     //&& s.RowKey.CompareTo(endTicks.ToString() + "_00000000-0000-0000-0000-000000000000") > 0
                     && s.PartitionKey == ("story_" + searchGuid)
                     ).Take(50);
        var query = stories as DataServiceQuery<Story>;
        var results = query.Execute();
        var response = results as QueryOperationResponse;

        Stories temp = new Stories();
        if(response.Headers.ContainsKey("x-ms-continuation-NextRowKey"))
        {
            temp.NextPartitionKey = response.Headers["x-ms-continuation-NextPartitionKey"];
            if (response.Headers.ContainsKey("x-ms-continuation-NextRowKey"))
            {
                temp.NextRowKey = response.Headers["x-ms-continuation-NextRowKey"];
            }
        }
        temp.List = results.ToList();

        return temp;
    }

Answer 1

cOK, I think there are a couple of things going on here. One I think there is a logic flaw. Shouldn't

Convert.ToInt64(s.RowKey.Substring(0, s.PartitionKey.IndexOf("_")))

be

Convert.ToInt64(s.PartitionKey.Substring(0, s.PartitionKey.IndexOf("_")))

Secondly you need to be very careful about which functions are supported by azure table queries. Generally they're not. I've tested .Substring() and .IndexOf() and they don't work in Azure Table queries, so the chances of .ToInt64() working is slim to none.

You might be able to reformat this to be

s => s.PartitionKey > startTicks.ToString() + "_"
&& s.PartitionKey < endTicks.ToString() + "_"
&& s.RowKey == "story_" + searchGuid

This will likely not generate a very efficient query because Azure can get confused if you have two filters based on partition key and just do a table scan. Another option is to not include the endTicks part of the query and when you process the results, when you get to one the partition key is greater than end ticks, stop processing the results.

Also your code as you have it written won't get all of the items based on the continuation token, it will just get the first set of results that are returned. I think your final code should look something like this (uncompiled, untested and I'm sure people can see some performance improvements:

private class ListRowsContinuationToken
{
    public string NextPartitionKey { get; set; }
    public string NextRowKey { get; set; }
}

public Stories SelectStory(DateTime start, DateTime end, string searchGuid)
{
    long startTicks = DateTime.MaxValue.Ticks - start.ToUniversalTime().Ticks;
    long endTicks = DateTime.MaxValue.Ticks - end.ToUniversalTime().Ticks;

var stories = _ServiceContext.CreateQuery<Story>("Story").Where(s => s.PartitionKey > startTicks.ToString() + "_"
                && s.PartitionKey < endTicks.ToString() + "_"
                && s.RowKey == "story_" + searchGuid).Take(50);

var query = stories as DataServiceQuery<Story>;

Stories finalList = new Stories();

var results = query.Execute();

ListRowsContinuationToken continuationToken = null;
bool reachedEnd = false;

do
{
    if ((continuationToken != null))
    {
        servicesQuery = servicesQuery.AddQueryOption("NextPartitionKey", continuationToken.NextPartitionKey);

        if (!string.IsNullOrEmpty(continuationToken.NextRowKey))
        {
            servicesQuery.AddQueryOption("NextRowKey", continuationToken.NextRowKey);
        }
    }

    var response = (QueryOperationResponse<T>)query.Execute();

    foreach (Story result in response)
    {
        if (result.PartitionKey < endTicks.ToString())
        {
            finalList.AddRange(result);
        }
        else
        {
            reachedEnd = true;
        }
    }

    if (response.Headers.ContainsKey("x-ms-continuation-NextPartitionKey"))
    {
        continuationToken = new ListRowsContinuationToken
        {
            NextPartitionKey = response.Headers["x-ms-continuation-NextPartitionKey"]
        };

        if (response.Headers.ContainsKey("x-ms-continuation-NextRowKey"))
        {
            continuationToken.NextRowKey = response.Headers["x-ms-continuation-NextRowKey"];
        }
    }
    else
    {
        continuationToken = null;
    }

} while (continuationToken != null && reachedEnd == false);

return finalList;

}