I wrote DbEntityImportEngine<TContext, TItem> to encapsulate the core processing of importing data into a database using Entity Framework. In general, importing various forms of data is eventually constrained to the database schema, validation rules, and in special cases custom logic. Additionally, importing is a general task that usually implies taking one or more items and committing them to a data source such as a database. I needed a way to abstract the fundamentals of importing data using Entity Framework that never change into reusable and extensible logic. This abstract class provides static core processing logic using a Queue<T> with a configurable backlog capacity before DbContext.SaveChanges is called. Other features include virtual methods that can be overridden to implement custom behavior by the deriving class, events for notification, and other useful properties such as being able to access queued items, validation exceptions, update exceptions, or unhandled exceptions.
The second major design goal I had in mind was of course performance. I didn't want to sacrifice the productivity Entity Framework ORM provides, so I spent a lot of time studying best practices to get the best performance possible from Entity Framework. When my goal is to achieve the best performance possible, I use the following rules when writing code.
DO NOT: use LINQ (Language Integrated Query). Use well-written traditional loops and iterators instead.
DO NOT: use lambda expressions. Use traditional methods instead.
DO: disable change tracking notification for DbContext or ObjectContext.
DO: manually set the entity key of a newly initialized entity and attach it to the DbContext or ObjectContext when you already know an entity's key information, instead of trying to get an instance of it from the database first.
DO: manually set the entity state of an entity or its individual properties instead of relying on change tracking notification.
CONSIDER: implementing a cache for repeated lookups or common information to prevent having to open connections and execute queries.
//-----------------------------------------------------------------------------
// <copyright file="DbEntityImportEngine`.cs" company="DCOM Productions">
// Copyright (c) DCOM Productions. All rights reserved.
// Written by David Anderson, DCOM Productions, Microsoft Partner (MSP)
//
// Licensed for public, read-only, reference use as a demonstration of how
// one might approach writing high-performance code to perform importing
// using the Entity Framework ORM.
// </copyright>
//-----------------------------------------------------------------------------
namespace DCOMProductions.Data.Entity.Import
{
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data.Entity;
using System.Data.Entity.Infrastructure;
using System.Data.Entity.Validation;
using System.Diagnostics;
using System.Linq;
/// <summary>
/// A generic import engine for importing items into a DbContext.
/// </summary>
/// <remarks>
/// This particular engine implementation uses a queue, and commits
/// changes in batches specified by the CommitBacklog property.
/// </remarks>
public abstract class DbEntityImportEngine<TContext, TItem> where TContext : DbContext
{
private Stopwatch internalClock;
private ICollection<Exception> unexpectedExceptions;
private ICollection<DbUpdateException> updateExceptions;
private ICollection<DbEntityValidationException> validationExceptions;
/// <summary>
/// Initializes a new instance of the DCOMProductions.Data.Entity.Import.DbEntityImportEngine class.
/// </summary>
/// <param name="settings">The importer settings to use.</param>
public DbEntityImportEngine()
{
internalClock = new Stopwatch();
CommitBacklog = 250;
Items = new Queue<TItem>();
}
/// <summary>
/// Occurs after DbContext.SaveChanges is called. This event occurs whether DbContext.SaveChanges was successful or unsuccessful.
/// </summary>
public event EventHandler<CommitEventArgs<TContext>> Commit;
/// <summary>
/// Occurs after there are no longer any items in the queue to import.
/// </summary>
public event EventHandler Complete;
/// <summary>
/// Occurs after an item has been imported into the current DbContext instance.
/// </summary>
public event EventHandler<ItemImportedEventArgs<TContext, TItem>> ItemImported;
/// <summary>
/// Occurs after an item has been imported to provide progress information.
/// </summary>
public event ProgressChangedEventHandler ProgressChanged;
/// <summary>
/// Occurs when an unexpected or unhandled exception occurs when importing an item.
/// </summary>
public event EventHandler<UnexpectedExceptionEventArgs<TContext>> UnexpectedException;
/// <summary>
/// Occurs when a DbUpdateException is thrown during DbContext.SaveChanges.
/// </summary>
public event EventHandler<UpdateExceptionEventArgs<TContext>> UpdateException;
/// <summary>
/// Occurs when there are one or more validation errors in the CommitBacklog prior to calling DbContext.SaveChanges.
/// </summary>
public event EventHandler<ValidationErrorsEventArgs<TContext>> ValidationErrors;
/// <summary>
/// Occurs when a DbEntityValidationException is thrown during DbContext.SaveChanges.
/// </summary>
public event EventHandler<ValidationExceptionEventArgs<TContext>> ValidationException;
/// <summary>
/// Gets or sets the number of entities to backlog before calling DbContext.SaveChanges.
/// </summary>
/// <remarks>The default value is 250.</remarks>
public int CommitBacklog
{
get;
set;
}
/// <summary>
/// Gets the current item being imported, if any.
/// </summary>
public TItem CurrentItem
{
get;
private set;
}
/// <summary>
/// Gets the elapsed time since DbEntityImportEngine.Run was called.
/// </summary>
public TimeSpan Elapsed
{
get
{
return internalClock.Elapsed;
}
}
/// <summary>
/// Gets the current queue of items.
/// </summary>
public Queue<TItem> Items
{
get;
private set;
}
/// <summary>
/// Gets the total number of items that were enqueued when the Run method was called.
/// </summary>
public Int32 ItemsStartCount
{
get;
private set;
}
/// <summary>
/// Gets a collection of exception's that were thrown during the import.
/// </summary>
public IEnumerable<Exception> UnexpectedExceptions
{
get
{
return unexpectedExceptions ?? (unexpectedExceptions = new HashSet<Exception>());
}
}
/// <summary>
/// Gets a collection of DbUpdateException exception's that were thrown during the import.
/// </summary>
public IEnumerable<DbUpdateException> UpdateExceptions
{
get
{
return updateExceptions ?? (updateExceptions = new HashSet<DbUpdateException>());
}
}
/// <summary>
/// Gets a collection of DbEntityValidationException exception's that were thrown during the import.
/// </summary>
public IEnumerable<DbEntityValidationException> ValidationExceptions
{
get
{
return validationExceptions ?? (validationExceptions = new HashSet<DbEntityValidationException>());
}
}
/// <summary>
/// Override to implement logic to return a new DbContext instance when requested.
/// </summary>
/// <returns>Returns a newly initialized DbContext.</returns>
protected abstract TContext CreateDbContext();
/// <summary>
/// Override to implement logic to handle any unhandled exceptions.
/// </summary>
/// <param name="ex">The unhandled exception object.</param>
/// <param name="context">The DbContext instance that caused the error.</param>
/// <param name="handled">Required by user code. Indicates whether user code was executed to handle the unexpected exception.</param>
/// <param name="actionTaken">Required by user code if <see cref="handled"/> is set to true. Describes the action that was taken.</param>
protected virtual void HandleUnexpectedException(System.Exception ex, TContext context, out Boolean handled, out String actionTaken)
{
handled = false;
actionTaken = String.Empty;
}
/// <summary>
/// Override to implement logic to handle any update exceptions that might occur.
/// </summary>
/// <param name="ex">The DbUpdateException that occurred.</param>
/// <param name="context">The DbContext instance that caused the error.</param>
/// <param name="handled">Required by user code. Indicates whether user code was executed to handle the update exception.</param>
/// <param name="actionTaken">Required by user code if <see cref="handled"/> is set to true. Describes the action that was taken.</param>
protected virtual void HandleUpdateException(DbUpdateException ex, TContext context, out Boolean handled, out String actionTaken)
{
handled = false;
actionTaken = String.Empty;
}
/// <summary>
/// Override to implement logic to handle any validation errors prior saving changes to the DbContext instance.
/// </summary>
/// <param name="validationErrors">The validation errors to handle.</param>
/// <param name="context">The DbContext instance that caused the error.</param>
/// <param name="handled">Required by user code. Indicates whether user code was executed to handle the validation errors.</param>
/// <param name="actionTaken">Required by user code if <see cref="handled"/> is set to true. Describes the action that was taken.</param>
protected virtual void HandleValidationErrors(IEnumerable<DbEntityValidationResult> validationErrors, TContext context, out Boolean handled, out String actionTaken)
{
handled = false;
actionTaken = String.Empty;
}
/// <summary>
/// Override to implement logic to handle any validation exceptions that might occur.
/// </summary>
/// <param name="ex">The DbEntityValidationException that occurred.</param>
/// <param name="context">The DbContext instance that caused the error.</param>
/// <param name="handled">Required by user code. Indicates whether user code was executed to handle the validation exception.</param>
/// <param name="actionTaken">Required by user code if <see cref="handled"/> is set to true. Describes the action that was taken.</param>
protected virtual void HandleValidationException(DbEntityValidationException ex, TContext context, out Boolean handled, out String actionTaken)
{
handled = false;
actionTaken = String.Empty;
}
/// <summary>
/// Override to implement logic to import the specified item.
/// </summary>
/// <param name="context">The DbContext instance to import the item into.</param>
/// <param name="item">The item to import.</param>
protected abstract void Import(TContext context, TItem item);
private void ImportLoop()
{
int itemsQueued = Items.Count;
for (int i = 0; i < itemsQueued; i += CommitBacklog)
{
using (TContext context = CreateDbContext())
{
for (int itemsDequeued = 0; itemsDequeued < CommitBacklog; itemsDequeued++)
{
if (Items.Count == 0)
{
break;
}
CurrentItem = Items.Dequeue();
try
{
Import(context, CurrentItem);
}
catch (Exception ex)
{
bool handled; string actionTaken;
HandleUnexpectedException(ex, context, out handled, out actionTaken);
RaiseUnexpectedException(ex, context, handled, actionTaken);
unexpectedExceptions.Add(ex);
}
finally
{
ImportedItem(context, CurrentItem);
RaiseItemImported(context, CurrentItem);
RaiseProgressChanged();
CurrentItem = default(TItem);
}
}
IEnumerable<DbEntityValidationResult> validationErrors = context.GetValidationErrors();
if (validationErrors.Count() > 0)
{
bool handled; string actionTaken;
HandleValidationErrors(validationErrors, context, out handled, out actionTaken);
RaiseValidationErrors(validationErrors, context, handled, actionTaken);
}
int committed = -1;
bool success = false;
try
{
committed = context.SaveChanges();
success = true;
}
catch (DbEntityValidationException ex)
{
bool handled; string actionTaken;
HandleValidationException(ex, context, out handled, out actionTaken);
RaiseValidationException(ex, context, handled, actionTaken);
validationExceptions.Add(ex);
}
catch (DbUpdateException ex)
{
bool handled; string actionTaken;
HandleUpdateException(ex, context, out handled, out actionTaken);
RaiseUpdateException(ex, context, handled, actionTaken);
updateExceptions.Add(ex);
}
catch (Exception ex)
{
bool handled; string actionTaken;
HandleUnexpectedException(ex, context, out handled, out actionTaken);
RaiseUnexpectedException(ex, context, handled, actionTaken);
unexpectedExceptions.Add(ex);
}
finally
{
RaiseCommit(context, committed, success);
}
}
}
}
/// <summary>
/// Override to implement logic after an item has been processed.
/// </summary>
/// <param name="context">The DbContext instance that was used when the item was processed.</param>
/// <param name="item">The item that was processed.</param>
protected virtual void ImportedItem(TContext context, TItem item)
{
}
/// <summary>
/// Override to implement any initialization logic.
/// </summary>
protected virtual void Initialize()
{
}
private void RaiseCommit(TContext context, int committed, bool success)
{
var handler = Commit;
if (handler != null)
{
handler(this, new CommitEventArgs<TContext>(context, committed, success));
}
}
private void RaiseComplete()
{
var handler = Complete;
if (handler != null)
{
handler(this, new EventArgs());
}
}
private void RaiseItemImported(TContext context, TItem item)
{
var handler = ItemImported;
if (handler != null)
{
handler(this, new ItemImportedEventArgs<TContext, TItem>(context, item));
}
}
private void RaiseProgressChanged()
{
var handler = ProgressChanged;
if (handler != null)
{
float a = Items.Count;
float b = ItemsStartCount;
int progressPercentage = 100 - Convert.ToInt32((a / b) * 100);
handler(this, new ProgressChangedEventArgs(progressPercentage, null));
}
}
private void RaiseUnexpectedException(Exception exception, TContext context, bool handled, string actionTaken)
{
var handler = UnexpectedException;
if (handler != null)
{
handler(this, new UnexpectedExceptionEventArgs<TContext>(exception, context, handled, actionTaken));
}
}
private void RaiseUpdateException(DbUpdateException exception, TContext context, bool handled, string actionTaken)
{
var handler = UpdateException;
if (handler != null)
{
handler(this, new UpdateExceptionEventArgs<TContext>(exception, context, handled, actionTaken));
}
}
private void RaiseValidationErrors(IEnumerable<DbEntityValidationResult> validationErrors, TContext context, bool handled, string actionTaken)
{
var handler = ValidationErrors;
if (handler != null)
{
handler(this, new ValidationErrorsEventArgs<TContext>(validationErrors, context, handled, actionTaken));
}
}
private void RaiseValidationException(DbEntityValidationException exception, TContext context, bool handled, string actionTaken)
{
var handler = ValidationException;
if (handler != null)
{
handler(this, new ValidationExceptionEventArgs<TContext>(exception, context, handled, actionTaken));
}
}
/// <summary>
/// Begins the import process.
/// </summary>
public void Run()
{
ItemsStartCount = Items.Count;
Initialize();
internalClock.Start();
ImportLoop();
internalClock.Stop();
RaiseComplete();
}
}
}