You could create a custom authorisation attribute inheriting from the standard AuthorizeAttribute with an optional bool parameter to specify whether authorisation is required or not.
public class OptionalAuthorizeAttribute : AuthorizeAttribute
{
 private readonly bool _authorize;
 public OptionalAuthorizeAttribute()
 {
  _authorize = true;
 }
 public OptionalAuthorizeAttribute(bool authorize)
 {
  _authorize = authorize;
 }
 protected override bool AuthorizeCore(HttpContextBase httpContext)
 {
  if(!_authorize)
   return true;
                    return base.AuthorizeCore(httpContext);
 }
}
Then you can decorate your base controller with that attribute:
[OptionalAuthorize]
public class ControllerBase : Controller
{
}
and for any controllers you don't want authorisation simply use the override with a 'false' - e.g.
[OptionalAuthorize(false)]
public class TestController : ControllerBase
{
 public ActionResult Index()
 {
  return View();
 }
}