Quantcast
Viewing all articles
Browse latest Browse all 445

Slow Record Loading into Teradata - with or without PK

I am trying to use the bulk load method using TDCommandBuilder in C# and also compare it to the individual insert statement performance using the following code. I move data back and forth between SQL Server and Teradata. I see speeds of several million rows per minute in SQL Server using sqlcommandbuilder versus about 50000 a minute in Teradata with  or without tdcommandbuilder. What am I doing wrong here? I am using Teradata driver version 14.0.0.0. Thank you.

                using (TdConnection connection = connect())        // initialize connection
                {


                    // set all required for batch elements and fill the schema
                    DataTable dtInsertBatch = new DataTable();
                    TdDataAdapter adapter = new TdDataAdapter();
                    adapter.SelectCommand = connection.CreateCommand();

                    adapter.SelectCommand.CommandText = "SELECT " + String.Join(", ", getColumnNames(loadData)) + " FROM " + tableName;

                    adapter.FillSchema(dtInsertBatch, SchemaType.Source);

                    if (null != _monitor) _monitor.onLogMessage(dtInsertBatch.Columns.Count + " columns populated into source schema using " + Environment.NewLine + adapter.SelectCommand.CommandText, "TeradataFastloadAdapter.insert");

                    if (0 == dtInsertBatch.PrimaryKey.Length)   // if no PK then set own insert command
                    {
                        if (null != _monitor) _monitor.onLogMessage("No primary key detected. Will insert using individual insert statements", "TeradataFastloadAdapter.insert");


                        TdCommand insertCommand = connection.CreateCommand();

                        insertCommand.CommandType = CommandType.Text;
                        insertCommand.CommandText = "INSERT INTO " + tableName + "(" +
                                                    String.Join(", ", getColumnNames(loadData)) + ")"
                                                    + " VALUES (" + String.Join(", ", getColumnNamesAsParameters(loadData)) +
                                                    ")";

                        foreach (DataColumn colParm in loadData.Columns)
                        {
                            // add new parameter
                            insertCommand.Parameters.Add(colParm.ColumnName, colParm.DataType);
                        }

                        if (null != _monitor) _monitor.onLogMessage("Starting insert into " + tableName, "TeradataFastloadAdapter.insert");
                        updateCount = 0;
                        // insert one by one
                        insertCommand.Transaction = connection.BeginTransaction();
                        foreach (DataRow row in loadData.Rows)
                        {
                            foreach (TdParameter parm in insertCommand.Parameters)
                            {
                                parm.Value = row[parm.ParameterName];
                            }
                            if (1 != insertCommand.ExecuteNonQuery())
                            {

                                insertCommand.Transaction.Rollback();
                                if (null != _monitor) _monitor.onLogMessage("Error: Individual statement number " + updateCount + "/" + loadData.Rows.Count + " failed to complete", "TeradataFastloadAdapter.insert");
                                throw new Exception("Unable to load all records using individual statement calls");
                            }
                            updateCount++;
                        }

                        insertCommand.Transaction.Commit();

                    }
                    else      // insert using builder when PK is present
                    {
                        if (null != _monitor) _monitor.onLogMessage("Primary key detected. Will use batch load to insert records", "TeradataFastloadAdapter.insert");

                        TdCommandBuilder builder = new TdCommandBuilder(adapter);
                        adapter.SelectCommand.Transaction = connection.BeginTransaction();

                        // add rows to insert
                        dtInsertBatch.Merge(loadData);

                        foreach (DataRow dataRow in dtInsertBatch.Rows)
                            if (dataRow.RowState == DataRowState.Unchanged) dataRow.SetAdded();

                        if (null != _monitor) _monitor.onLogMessage("Starting batch insert into " + tableName, "TeradataFastloadAdapter.insert");

                        updateCount = adapter.Update(dtInsertBatch);

                        if (updateCount != dtInsertBatch.Rows.Count)
                        {
                            if (null != _monitor) _monitor.onLogMessage("Error: Expected insert count is " + loadData.Rows.Count + " but actual is " + updateCount, "TeradataFastloadAdapter.insert");
                            throw new Exception("Unable to load all records using batch load");
                        }
                        else
                        {
                            if (null != _monitor) _monitor.onLogMessage("Batch insert successfully inserted " + updateCount + " rows", "TeradataFastloadAdapter.insert");
                        }

                        adapter.SelectCommand.Transaction.Commit();
                    }
                }

 

Forums: 

Viewing all articles
Browse latest Browse all 445

Trending Articles