I have a macro that imports a worksheet and compares it for duplicates in the current worksheet before copying the data.
It works for small data sets but freezes when ran on the workbook I created it for as it has a very large data set (100,000 rows+).
Need help optimising my code or fixing this please.
Any help is greatly appreciated
It works for small data sets but freezes when ran on the workbook I created it for as it has a very large data set (100,000 rows+).
Need help optimising my code or fixing this please.
Any help is greatly appreciated
VBA Code:
Sub DownloadImportAndCheckDuplicates()
'
Dim fileToOpen As Variant
Dim fileFilterPattern As String
Dim wsMaster As Worksheet
Dim wbTextImport As Workbook
Dim dlr As Long
Dim lr As Long
Dim lImpC As Long
Dim lImpR As Long
Dim countMatch As Boolean
Dim concat As Collection
Set concat = New Collection
Application.ScreenUpdating = False
Set wsMaster = ThisWorkbook.Worksheets("Download Data")
With wsMaster
lr = .Cells(Rows.Count, 1).End(xlUp).Offset(1).Row
dlr = .Cells(Rows.Count, 1).End(xlUp).Row
' adds date downloaded, asset name & email to collection
For i = 5 To dlr
concat.Add .Cells(i, 1) & "__" & .Cells(i, 2) & "__" & .Cells(i, 3)
Next i
End With
fileFilterPattern = "Microsoft Excel Workbooks (*.xls*),*.xls*"
fileToOpen = Application.GetOpenFilename(fileFilterPattern)
' open workbook
If fileToOpen = False Then
' input Cancelled
MsgBox "No file Selected."
Else
Workbooks.OpenText _
Filename:=fileToOpen, _
StartRow:=2, _
DataType:=xlDelimited, _
Tab:=True
Set wbTextImport = ActiveWorkbook
' limpC last column with data
' limmpR last row with data
With wbTextImport.Worksheets(1)
lImpC = .Cells(1, .Columns.Count).End(xlToLeft).Column
lImpR = .Cells(Rows.Count, 1).End(xlUp).Row
arrdata = .Range("A1:C" & lImpR).Value
End With
End If
countMatch = False
Dim j As Long
' nested loop comparing the concatenated string from asset upload data and 3d array of imported text
For i = LBound(arrdata) To UBound(arrdata) ' I'm assuming the data copied has headers, if not, change 2 for 1
For j = 1 To concat.Count
'Debug.Print concat(j) & " concat"
'Debug.Print (arrdata(i, 1) & "__" & arrdata(i, 2) & "__" & arrdata(i, 3)) & " toCompare"
If concat(j) = (arrdata(i, 1) & "__" & arrdata(i, 2) & "__" & arrdata(i, 3)) Then
MsgBox "Duplicates found, please check data you are attempting to copy"
countMatch = True
GoTo LastLine
Else
' If no duplicates import worksheet from a2 to last cells with data on this worksheet from lr in c
If countMatch = False And i = UBound(arrdata) Then
With wbTextImport.Worksheets(1)
.Range("A2", .Cells(lImpR, lImpC)).Copy wsMaster.Range("A" & lr)
End With
End If
End If
Next j
Next i
LastLine:
Debug.Print "We're out"
wbTextImport.Close False
Application.ScreenUpdating = True
'
End Sub