GAE: Why does this code crash the devserver?

Question

I'm using GAE 1.3.5 devserver SDK with Python. When I uncomment this line of code, GAEUnit hangs every time I try to run my test suite:

dep_arc_tail_q = db.GqlQuery("SELECT * FROM DependencyArcTail WHERE courses = :1", course)

#problem line
modelutils.applyToResultsOfQuery(lambda tails : modelutils.removeCourseFromTails(course, tails), dep_arc_tail_q)

The modelutils methods:

def applyToResultsOfQuery(func, q):
    results = q.fetch(1000)
    while results:
        func(results)
        results = q.fetch(1000)

def removeCourseFromTail(course, tail):
    # tail = DependencyArcTail.get(key_tail)
    if not course in tail.courses:
        return

    if len(tail.courses) == 1:
        DependencyArcTail.delete(tail)
        return

    tail.courses.remove(course)

def removeCourseFromTails(course, tails):
    ''' Removes `course` from a collection of `tails` '''
    removeThisCourseFromTail = functools.partial(removeCourseFromTail, course)
    map(removeThisCourseFromTail, tails)

I'm not getting any sort of crash or traceback... the devserver is just entirely unresponsive.

Notably, another function also uses modelutils.applyToResultsOfQuery to delete models:

def _deleteType(kind):
 q = kind.all(keys_only=True)
 deleteResultsOfQuery(q)

def deleteResultsOfQuery(q):
    applyToResultsOfQuery(db.delete, q)

The tests run fine with these methods, making me think that the problem isn't with applyToResultsOfQuery.

Here are the models in use:

class Course(db.Model):
    dept_code = db.StringProperty(required=True)
    number = db.IntegerProperty(required=True)
    title = db.StringProperty(multiline=True) # not sure that this should be multiline...
    pickled_pre_reqs = db.StringProperty(multiline=True)

    # the unparsed pre req phrase (like "CS 2110 or 1110")
    unparsed_pre_reqs = db.StringProperty()

    # the full value of the Note part of the course catalog listing
    full_description = db.TextProperty()

    # the page on which this course is described
    course_catalog_url = db.LinkProperty()

    parse_succeeded = db.BooleanProperty()

    # true if this course has had its dependency graph built
    graph_built = db.BooleanProperty()

    tailsMemberOf = db.ListProperty(db.Key)

    def getPreReqs(self):
        return pickle.loads(str(self.pickled_pre_reqs))

    def __repr__(self):
        if self.dept_code == "UNKNOWN":
            return "Unknown course"

        return "%s %s: %s" % (self.dept_code, self.number, self.title)

    # just use __dict__?
    def __attrs(self):
        return (self.dept_code, self.number, self.title, self.pickled_pre_reqs, self.full_description, self.unparsed_pre_reqs, self.course_catalog_url)

    def __eq__(self, other):
        return isinstance(other, Course) and self.__attrs() == other.__attrs()

    def __hash__(self):
        return hash(self.__attrs())

class DependencyArcTail(db.Model):
    ''' A list of courses that is a pre-req for something else '''
    courses = db.ListProperty(db.Key) # can this be changed to Course.key?

    ''' a list of heads that reference this one '''
    forwardLinks = db.ListProperty(db.Key)

    def __repr__(self):
        return "DepArcTail %d: courses='%s' forwardLinks='%s'" % (id(self), getReprOfKeys(self.courses), getIdOfKeys(self.forwardLinks))

    def __eq__(self, other):
        return isinstance(other, DependencyArcTail) and set(self.courses) == set(other.courses) and set(self.forwardLinks) == set(other.forwardLinks)

    def __hash__(self):
        return hash((frozenset(self.courses), frozenset(self.forwardLinks)))

What else could I be doing wrong here?

UPDATE: It appears that after running the test with that line uncommented, the entire dev server crashes. If I navigate to a non-test page after that, I get 500'd. I'm not sure what conclusions to draw from that.

UPDATE 2: If I get rid of modeutils, and write it out another way, it works fine:

dep_arc_tail_q = db.GqlQuery("SELECT * FROM DependencyArcTail WHERE courses = :1", course)
# modelutils.applyToResultsOfQuery(lambda tails : modelutils.removeCourseFromTails(course, tails), dep_arc_tail_q)
results = dep_arc_tail_q.fetch(1000)
# while results:
for tail in results:
    modelutils.removeCourseFromTail(course, tail)
    # results = dep_arc_tail_q.fetch(1000)

However, if I change the comments up, it fails again:

dep_arc_tail_q = db.GqlQuery("SELECT * FROM DependencyArcTail WHERE courses = :1", course)
# modelutils.applyToResultsOfQuery(lambda tails : modelutils.removeCourseFromTails(course, tails), dep_arc_tail_q)
results = dep_arc_tail_q.fetch(1000)
while results:
    for tail in results:
        modelutils.removeCourseFromTail(course, tail)
    results = dep_arc_tail_q.fetch(1000)

Am I getting an infinite loop? Is fetch() not working as I expect?

Answer 1

fetch(1000) will always return the first 1000 results.

You may want to try using cursors, something like this:

results = dep_arc_tail_q.fetch(1000) 
while results: 
    for tail in results: 
        modelutils.removeCourseFromTail(course, tail) 
    results = dep_arc_tail_q.with_cursor(dep_arc_tail_q.cursor()).fetch(1000)