I recently was given an assignment where the enterprise decided to access their legacy databases by using Windows Communication Foundation (WCF). So in populating the data warehouse I had to retrieve the legacy data using the web service. If WSHttpBinding or BasicHttpBinding had been selected, using the web service with SSIS would not have been a problem. I could have used a Web Service Task, or a Script Task, or a Script Component. As a matter of fact, here is a blog that provides more details on "How to Configure an SSIS Package to Access a Web Service using WCF" written by David Browne. But I had to use NetTcpBinding. With NetTcpBinding, I could not get the Web Service Task to work and I had a lot of problems in getting the SSIS Script Component to work.
After struggling with this issue for a while (I kept getting errors indicating that there was a problem with the endpoint address), one of the .NET developers modified the code in my SSIS Script Component so that I could use the web service. Here is what I ended up with.
I setup a Data Flow Task and added a Script Component with 7 outputs.
In the Script Component I added a service reference to the WCF service (named DataWarehouse.svc).
The .NET developer modified the PreExecute and PostExecute methods in the Script Component (Namespaces was also modified) so that it looks like the following.
| #region Namespaces using System; using System.Configuration; using System.Data; using Microsoft.SqlServer.Dts.Pipeline.Wrapper; using Microsoft.SqlServer.Dts.Runtime.Wrapper; using System.ServiceModel; using System.ServiceModel.Description; using System.Xml; using SC_d54ab3221c374499978fe37e7b2f2256.DataWarehouseServiceReference; #endregion [Microsoft.SqlServer.Dts.Pipeline.SSISScriptComponentEntryPointAttribute] public class ScriptMain : UserComponent { ChannelFactory channelFactory; IDataWarehouseServices client; public override void PreExecute() { var binding = new NetTcpBinding(SecurityMode.TransportWithMessageCredential, true); //configure the binding binding.CloseTimeout = TimeSpan.FromMinutes(1); binding.OpenTimeout = TimeSpan.FromMinutes(1); binding.ReceiveTimeout = TimeSpan.FromMinutes(1); binding.SendTimeout = TimeSpan.FromMinutes(1); binding.TransactionFlow = false; binding.TransferMode = TransferMode.Buffered; binding.TransactionProtocol = TransactionProtocol.OleTransactions; binding.HostNameComparisonMode = HostNameComparisonMode.StrongWildcard; binding.ListenBacklog = 10; binding.MaxBufferPoolSize = 524288; binding.MaxBufferSize = 65536000; binding.MaxConnections = 10; binding.MaxReceivedMessageSize = binding.MaxBufferSize; binding.ReliableSession = new OptionalReliableSession() {Enabled = false, Ordered = true, InactivityTimeout = TimeSpan.FromMinutes(10)}; // Security binding.Security.Mode = SecurityMode.Transport; binding.Security.Transport.ClientCredentialType = TcpClientCredentialType.Windows; binding.Security.Transport.ProtectionLevel = System.Net.Security.ProtectionLevel.EncryptAndSign; binding.Security.Message.ClientCredentialType = MessageCredentialType.Windows; var endpointAddress = new EndpointAddress(new Uri(@"net.tcp://msappdev/domainservices/DataWarehouse.svc"), EndpointIdentity.CreateUpnIdentity("app_agldev_service@agl.com")); channelFactory = new ChannelFactory(binding, endpointAddress); foreach (OperationDescription op in channelFactory.Endpoint.Contract.Operations) { DataContractSerializerOperationBehavior behavior = op.Behaviors.Find() as DataContractSerializerOperationBehavior; if (behavior != null) behavior.MaxItemsInObjectGraph = 100000000; } //create the channel client = channelFactory.CreateChannel(); } public override void PostExecute() { //base.PostExecute(); //close the channel IClientChannel channel = (IClientChannel)client; channel.Close(); //close the ChannelFactory channelFactory.Close(); } |
Then I modified the CreateNewOutputRows so that it looks like the following.
| public override void CreateNewOutputRows() { string mainPolicyId = String.Empty; foreach (var id in client.GetAllPolicyIds()) { Policy pol = client.GetPolicy(id, PolicyWith.All); if (pol != null) { polOutputBuffer.AddRow(); mainPolicyId = id.Num.ToString(); polOutputBuffer.id = mainPolicyId; if (pol.Reinsurances != null { foreach (ReInsurance reins in pol.Reinsurances) { reinsOutputBuffer.AddRow(); reinsOutputBuffer.streamno = reins.Id; reinsOutputBuffer.policyid = mainPolicyId; if (reins.Earnings != null { foreach (Earning ern in reins.Earnings) { if (pol.CurrentExchangeRate == 0) { earnOutputBuffer.AddRow(); earnOutputBuffer.streamno = ern.EarningId.StreamId; } else { fcEarnOutputBuffer.AddRow(); fcEarnOutputBuffer.streamno = ern.EarningId.StreamId; } } } } } if (pol.Premiums != null) { foreach (Premium prem in pol.Premiums) { premOutputBuffer.AddRow(); premOutputBuffer.policyid = mainPolicyId; premOutputBuffer.premctgr = prem.PremiumId.Category.ToString(); if (prem.CashTransaction != null) { premOutputBuffer.CashTransKey = prem.CashTransaction.Id; cashTransOutputBuffer.AddRow(); cashTransOutputBuffer.CashTransKey = prem.CashTransaction.Id; } if (prem.PolicyCashs != null) { foreach (PolicyCash pc in prem.PolicyCashs) { policyCashOutputBuffer.AddRow(); policyCashOutputBuffer.policyId = mainPolicyId; } } } } } } } } |
And everything worked as I had hoped.