How can I use multiple databases in the same request in Cherrypy and SQLAlchemy?

Question

Hey SO'ers,

My app connects to multiple databases using a technique similar to this. It works so long as I don't try to access different databases in the same request. Having looked back to the above script I see they have written a comment to this end:

SQLAlchemy integration for CherryPy,
such that you can access multiple databases,
but only one of these databases per request or thread.

My app now requires me to fetch data from Database A and Database B. Is it possible to do this in a single request?

Please see below for sources and examples:

Working Example 1:

from model import meta

my_object_instance = meta.main_session().query(MyObject).filter(
    MyObject.id == 1
).one()

Working Example 2:

from model import meta

my_user = meta.user_session().query(User).filter(
    User.id == 1
).one()

Error Example:

from model import meta

my_object_instance = meta.main_session().query(MyObject).filter(
    MyObject.id == 1
).one()

my_user = meta.user_session().query(User).filter(
    User.id == 1
).one()

This errors with:

(sqlalchemy.exc.ProgrammingError) (1146, "Table 'main_db.user' doesn't exist")

Sources:

# meta.py
import cherrypy
import sqlalchemy
from sqlalchemy import MetaData
from sqlalchemy.orm import scoped_session, sessionmaker
from sqlalchemy.ext.declarative import declarative_base

# Return an Engine
def create_engine(defaultschema = True, schema = "", **kwargs):

    # A blank DB is the same as no DB so to specify a non-schema-specific connection just override with defaultschema = False
    connectionString = 'mysql://%s:%s@%s/%s?charset=utf8' % (
        store['application'].config['main']['database-server-config-username'],
        store['application'].config['main']['database-server-config-password'],
        store['application'].config['main']['database-server-config-host'],
        store['application'].config['main']['database-server-config-defaultschema'] if defaultschema else schema
    )
    # Create engine object. we pass **kwargs through so this call can be extended
    return sqlalchemy.create_engine(connectionString, echo=True, pool_recycle=10, echo_pool=True, encoding='utf-8', **kwargs)

# Engines
main_engine = create_engine()
user_engine = None

# Sessions
_main_session = None
_user_session = None

# Metadata
main_metadata = MetaData()
main_metadata.bind = main_engine
user_metadata = MetaData()

# No idea what bases are/do but nothing works without them
main_base = declarative_base(metadata = main_metadata)
user_base = declarative_base(metadata = user_metadata)

# An easy collection of user database connections
engines = {}

# Each thread gets a session based on this object
GlobalSession = scoped_session(sessionmaker(autoflush=True, autocommit=False, expire_on_commit=False))

def main_session():
    _main_session = cherrypy.request.main_dbsession
    _main_session.configure(bind=main_engine)

    return _main_session

def user_session():
    _user_session = cherrypy.request.user_dbsession
    _user_session.configure(bind = get_user_engine())

    return _user_session

def get_user_engine():

    # Get dburi from the users instance
    dburi = cherrypy.session['auth']['user'].instance.database

    # Store this engine for future use
    if dburi in engines:
        engine = engines.get(dburi)
    else:
        engine = engines[dburi] = create_engine(defaultschema = False, schema = dburi)

    # Return Engine
    return engine


def get_user_metadata():
    user_metadata.bind = get_user_engine()
    return user_metadata

# open a new session for the life of the request
def open_dbsession():
    cherrypy.request.user_dbsession = cherrypy.thread_data.scoped_session_class
    cherrypy.request.main_dbsession = cherrypy.thread_data.scoped_session_class
    return

# close the session for this request
def close_dbsession():
    if hasattr(cherrypy.request, "user_dbsession"):
        try:
            cherrypy.request.user_dbsession.flush()
            cherrypy.request.user_dbsession.remove()
            del cherrypy.request.user_dbsession
        except:
            pass
    if hasattr(cherrypy.request, "main_dbsession"):
        try:
            cherrypy.request.main_dbsession.flush()
            cherrypy.request.main_dbsession.remove()
            del cherrypy.request.main_dbsession
        except:
            pass

    return

# initialize the session factory class for the selected thread
def connect(thread_index):
    cherrypy.thread_data.scoped_session_class = scoped_session(sessionmaker(autoflush=True, autocommit=False))
    return

# add the hooks to cherrypy
cherrypy.tools.dbsession_open = cherrypy.Tool('on_start_resource', open_dbsession)
cherrypy.tools.dbsession_close = cherrypy.Tool('on_end_resource', close_dbsession)
cherrypy.engine.subscribe('start_thread', connect)

Answer 1

You could also choose an ORM that is designed from the ground up for multiple databases, like Dejavu.