VBA Web Scraping - FindElementBy - DownloadCSV Button ElementNotVisibleError Runtime error 11

[johnd3099]

Option Explicit

Sub Download()
    Dim ChromeLocation As String
    ChromeLocation = "C:\Program Files\Google\Chrome\Application\chrome.exe" 'Location of Chrome.exe in your PC
   
    Dim cd As Selenium.ChromeDriver
    Set cd = New Selenium.ChromeDriver
    cd.SetBinary ("C:\Program Files\Google\Chrome\Application\chrome.exe")
   
    cd.Wait (3000)
   
    Dim FindBy As New Selenium.By
    Dim ResultSections As Selenium.WebElements
    Dim ResultSection As Selenium.WebElement
    Dim MyElement   As Object
    Dim SearchInput As Selenium.WebElement
    Dim SearchButton As Selenium.WebElement

    Call ClearImmediateWindow
   
'Genuine Stockbroker Username & Password        The ony way I could think of for someone to help solve my download problem
    Dim url As String
    url = "https://www.cmcmarketsstockbroking.com.au/"
   
    Dim UserName As String
    Dim Password As String
    UserName = "*********.com.au"                         'I  can provide these if requested
    Password = ".........."
   
    Dim DownloadURL As String
    DownloadURL = "https://www.cmcmarketsstockbroking.com.au/Manage/Watchlist/6213838?t=Watchlist"
'Genuine

    cd.AddArgument "start-maximized"
   
    cd.Start baseURL:=url
    cd.get "/", False
   
'    cd.Window.SetSize 1850, 1000
'    cd.Window.SetPosition 0, 0

'<input type="text" placeholder="Enter your username" id="logonAccount" name="logonAccount" class="sc-iktFfs bkQKJy sc-jQbJRc guveaB" value> == $0
    cd.FindElementById("logonAccount").Click
    cd.SendKeys (UserName)
   
'<input placeholder="Enter your password" id="logonPassword" name="logonPassword" class="sc-iktFfs bkQKJy sc-jUEmfL fsdOLa" type="password"> == $0
    cd.FindElementById("logonPassword").Click
    cd.SendKeys (Password)
   
'<button label="Login" type="submit" id="loginButton" class="sc-iqHYmW imzyTg sc-kIeSZW uCDoN"><span class="sc-gKseQn dhQxyI">Login</span></button>
    cd.FindElementById("loginButton").Click
   
  Debug.Print "Logged in to CMC"
 
'*****************************   AT THIS POINT,   WE HAVE LOGGED IN    ******************************************

    cd.get DownloadURL, False   'so far,  so good,  we have MarketCap1 page open,  wanting to press the Download CSV button
   
    If 1 = 1 Then               'This code is here to see .ResponseText,   because I've tried everything to Download
                                        'Change 'if 1 <> 1'  to  'if 1 = 1" to see the ResponseText in Sheet1        No longer applicable
        Dim XMLReq As Object
        Dim XMLHTTP As Object
        Set XMLHTTP = CreateObject("MSXML2.serverXMLHTTP")
       
        '<input type="button" class="inputbutton long btn" ng-click="ctrl.downloadCsv()" value="Download CSV">
        Dim HTMLDoc As MSHTML.HTMLDocument
        Dim HTMLDoc2 As MSHTML.HTMLDocument
        Dim LinkS As MSHTML.IHTMLElementCollection
        Dim Link As MSHTML.IHTMLElement
        Dim DownloadCSV As MSHTML.IHTMLElement
        Dim FileLinkFound As Boolean
        Dim FileURL As String
        Dim ReqStatus As Long
        Dim ResponseText As String
        Dim Http As New MSXML2.XMLHTTP60
   
        Set Http = CreateObject("MSXML2.XMLHTTP")
        With Http
            .Open "GET", DownloadURL, False
            .send (DownloadURL)
            Debug.Print .ResponseText
        End With
   
    End If

'Element        <input type="button" class="inputbutton long btn" ng-click="ctrl.downloadCsv()" value="Download CSV">
'xpath          //*[@id="autorefreshswitch"]/div/input
'css            #autorefreshswitch > div > input
'Full xpath     /html/body/form/div[4]/div[3]/div/div/div/div[4]/div[2]/div/input
    If 1 = 1 Then
        If Not cd.IsElementPresent(FindBy.Class("inputbutton")) Then
            Debug.Print "Could not find 'inputbutton'"
            Exit Sub
        End If
        Set SearchInput = cd.FindElementByClass("inputbutton")
        cd.SendKeys ("inputbutton")
   
        If Not cd.IsElementPresent(FindBy.Class("long")) Then
            Debug.Print "Could not find 'long'"
            Exit Sub
        End If
        Set SearchInput = cd.FindElementByClass("long")
        cd.SendKeys ("long")
       
        If Not cd.IsElementPresent(FindBy.Class("btn")) Then
            Debug.Print "Could not find 'btn'"
            Exit Sub
        End If
        Set SearchInput = cd.FindElementByClass("btn")
        cd.SendKeys ("btn")                                     'GOOD up to here
   
    'Narrowing the scope
        Dim OtdListItems As Selenium.WebElements
        Dim OtdListItem As Selenium.WebElement
        Dim OtdDiv As Selenium.WebElement
        Dim OtdLists As Selenium.WebElements
        Dim OtdList1 As Selenium.WebElement

        Set OtdDiv = cd.FindElementByXPath("//*[@id=""autorefreshswitch""]/div/input")       'did not bomb,  but no download
        Debug.Print OtdDiv.Text
        'Finding ng-click
        'xpath="//*[@id=""autorefreshswitch""]/div/input")
       
        Set OtdLists = cd.FindElementsByXPath("//*[contains(text(),' ng-click')]")
        If OtdLists Is Nothing Then
            Debug.Print "' ng-click' not found"
        Else
            Debug.Print "' ng-click' FOUND,  trying to CLICK"
            Debug.Print OtdLists.Count                                  'OtdLists.Count is ZERO
        End If

        cd.FindElementsByXPath ("//*[contains(text(),' ng-click')]")    'Does not BOMB
       
        For Each OtdList1 In OtdLists
            Debug.Print OtdList1.Text           'Nothing
        Next OtdList1

    End If
       
    '<input type="button" class="inputbutton long btn" ng-click="ctrl.downloadCsv()" value="Download CSV">
    cd.FindElementByClass ("btn")
    cd.FindElementByClass("btn").Click      'BOMBS here     ElementNotVisibleError  Runtime error 11
   
   
    'Don't know where to go from here ???
   
   
'---------------------------------------------------------------------------------------------------------------------------------------------

    'The following CODE is all stuff that I've tried
    Dim myCount As Long
    myCount = 0
   
    If 1 <> 1 Then
    'Find elements by tag
        Dim H2Headers As Selenium.WebElements
        Dim H2Header As Selenium.WebElement
        Dim Headlines As Selenium.WebElements
        Dim Headline As Selenium.WebElement
   
        'Set H2Headers = cd.FindElementsByXPath("//*[@id='autorefreshswitch']/div/input")
        'Set H2Headers = cd.FindElementsByXPath("ng-click=""ctrl.downloadCsv()""")
       
        cd.FindElementsByCss("#autorefreshswitch > div > input").Click
   
        Debug.Print "H2Headers.Count = " & H2Headers.Count
        If H2Headers.Count = 0 Then
            Debug.Print "No H2 headers found ... exitting"
            Exit Sub
        End If
   
        For Each H2Header In H2Headers
        myCount = myCount + 1
            Debug.Print myCount & "  H2Header.tagname = " & H2Header.tagname & "H2Header.Text = " & H2Header.Text
            Debug.Print H2Header.tagname & "   " & H2Header.Text
        Next H2Header
       
        H2Header(1).Click
        'H2Headers(1).submit
    End If
   
'Element        <input type="button" class="inputbutton long btn" ng-click="ctrl.downloadCsv()" value="Download CSV">
'xpath          //*[@id="autorefreshswitch"]/div/input
'css            #autorefreshswitch > div > input
'Full xpath     /html/body/form/div[4]/div[3]/div/div/div/div[4]/div[2]/div/input
   
    If 1 <> 1 Then
        If 1 <> 1 Then

            If Not cd.IsElementPresent(FindBy.XPath("//*[@id='autorefreshswitch']/div/input")) Then
                Debug.Print "element not present"
            End If
           
            While cd.Busy Or cd.ReadyState <> 4: DoEvents: Wend

            'cd.FindElementByXPath("//*[@id=""autorefreshswitch""]/div/input").submit       'StaleElementReferenceError
            cd.FindElementByXPath("//*[@id=""autorefreshswitch""]/div/input").Click         'did not bomb,  but did not download
           
        End If
       
        'css    #autorefreshswitch > div > input
        If 1 <> 1 Then
            'Set OtdDiv = cd.FindElementByCss("#autorefreshswitch")
            cd.FindElementByXPath("//*[@id=""autorefreshswitch""]/div/input").Click       'did not bomb,  but did not download
            'Set OtdDiv = cd.FindElementByXPath("//*[@id=""autorefreshswitch""]/div/input")
            'Set OtdDiv = cd.FindElementByClass("inputbutton")
            Debug.Print "#autorefreshswitch = " & OtdDiv.Text
            If OtdDiv.Text = "" Then
                Debug.Print "No mp-otd headers found ... exitting"
                'Exit Sub
            End If
           
            Set OtdLists = cd.FindElementsByClass("long")
            Debug.Print "div.Count = " & OtdLists.Count
            If OtdLists.Count = 0 Then
                Debug.Print "No ul headers found ... exitting"
                'Exit Sub
            End If
           
            Set OtdList1 = cd.FindElementByClass("btn")
            cd.FindElementByClass("btn").Click
            Debug.Print "input.Count = " & OtdLists.Count
            If OtdLists.Count = 0 Then
                Debug.Print "No ul headers found ... exitting"
                'Exit Sub
            End If
           
            For Each OtdList1 In OtdLists
                Debug.Print OtdList1.Text
            Next OtdList1

        End If
    End If
End Sub 'Download


Sub ClearImmediateWindow()
    If 1 <> 1 Then      'Not always working
        Application.SendKeys ("^g")            'Gets into Immediate window
       
        Application.SendKeys "^g^a"               'Select ALL in Immediate window
        Application.SendKeys "{BACKSPACE}"      'Send the DEL key
       
        Application.SendKeys "+{F7}"        'Gets into Code window
    End If
End Sub

 

[johnd3099]

You have a private message
The htm code start with the warning
"Your older web browser is no longer supported and some features of the platform, including online trading, will not work. We recommend upgrading your browser to the latest version of Chrome, Firefox or Internet Explorer"
So maybe it would be wise if you upgrade your Chrome (that will require updating also the selenium driver)

The code generated some "debug.print" messages, could you read that information and put it in your next message?

EDIT:
I was not allowed to send a pm to you, so "you don't have a pm"

I logged in using FireFox, also Brave, and both OuterHtml show that warning

 

[Abdmujib]

You will see a message box icon, press it and type the username of the person you want to send the message using "@"

 

[Peter_SSs]

A gentle reminder to all contributors in this thread that the thread questions/answers/information should be kept in the public forum, not taken to the conversation message system - refer to #4 of the Forum Rules

 

[Anthony47]

@Peter_SSs I tried sending a pm to the OP to advise that some confidential info was embedded into the Outerhtm text file he had shared; then I reported the post to a Moderator that removed the link

Having created a test account I was able to get to this macro:

Sub Download2()
Dim CD As Selenium.ChromeDriver
Dim tObj As Object, Url As String, I As Long, myFile
Dim UserName As String
Dim Password As String
Dim DownloadURL As String
Dim dRequest As Date, dPath As String
Dim fDnT As Date, fOk As Boolean
'
Url = "https://www.cmcmarketsstockbroking.com.au/"
DownloadURL = https://www.cmcmarketsstockbroking.com.au/Manage/Watchlist/6213838?t=Watchlist
'
dPath = "C:\PROVA\"                             '<<< The destination path
UserName = "*********"                          '<<< Username & Password
Password = "*********"
'
Set CD = New Selenium.ChromeDriver
'
'Preferences:
CD.SetPreference "download.default_directory", dPath
CD.SetPreference "download.directory_upgrade", True
CD.SetPreference "download.prompt_for_download", False
'
Debug.Print ">>>> Start"
CD.Start baseURL:=Url
CD.get "/"
'
'Wait for the Account inputbox:
For I = 1 To 20
    Set tObj = CD.FindElementsById("logonAccount")
    CD.Wait 500
    If tObj.Count > 0 Then Exit For
Next I
If I > 20 Then
    Debug.Print "Failed to login_1"
    MsgBox ("Unable to login")
    Exit Sub
End If
Debug.Print "Searched Account Inputbox", I
tObj(1).SendKeys (UserName)
'...then send Password
CD.FindElementsById("logonPassword")(1).SendKeys (Password)
'...and click Login:
CD.FindElementById("loginButton").Click
Debug.Print "Login clicked"
'Wait for landing Url:
For I = 1 To 20
    CD.Wait 500
    If InStr(1, CD.Url, ".au/clients/homepage", vbTextCompare) > 0 Then Exit For
Next I
If I > 20 Then
    Debug.Print "Failed to login_2"
    MsgBox ("Unable to login")
    Exit Sub
End If
Debug.Print "Landed?", I, CD.Url
'
'open the watchlist page:
CD.get DownloadURL, False   'so far,  so good,  we have MarketCap1 page open,  wanting to press the Download CSV button
'
'Wait & click the button:
For I = 1 To 20
    CD.Wait 500
    Set tObj = CD.FindElementsByCss("input[class='inputbutton long btn']")
    If tObj.Count > 0 Then Exit For
Next I
Debug.Print "i=" & I, "Count=" & tObj.Count
If I < 20 Then
    dRequest = Now
    CD.Wait 1000
    tObj(1).Click
    Debug.Print "Clicked"
Else
    MsgBox ("Button not catched")
    Debug.Print "Button not catched"
    Stop                                '<<< For Debug
    Exit Sub
End If
'
'Look into the destination path:
For I = 1 To 20                         'Max 10 secs
    CD.Wait 500
    myFile = Dir(dPath & "*.csv")
    Do
        If myFile = "" Then Exit Do
        fDnT = FileDateTime(dPath & myFile)
        If fDnT > dRequest Then
            fOk = True                  'eg download executed
        End If
        If fOk Then Exit Do
        myFile = Dir
    Loop
    If fOk Then Exit For
Next I
If fOk Then
    Debug.Print "Downloaded", I, dPath & myFile
    MsgBox ("Downloaded: " & dPath & myFile)
Else
    Debug.Print "Download failure", I
    MsgBox ("No file has been downloaded :-(")
End If

End Sub

The lines marked <<< need to be customized according the comments.
Most of the code is to check and trace the operations and print some debug information to the Immediate window.
At the beginning of the macro there is the definition of which is the downloading path.

After having clicked the download button I check the destination path and wait for the file, before terminating the macro (whereas ending the macro earlier whould delete the selenium driver and kill any ongoing downloading)

Try...

 

[Peter_SSs]

@Peter_SSs I tried sending a pm to the OP to advise that some confidential info was embedded into the Outerhtm text file he had shared;

OK, thanks for the clarification.

 

[johnd3099]

Hello Anthony,

The download works. Many thanks to you and Peter_SSs.

I had to use my code to log in with username/password, as this code "tObj(1).SendKeys (UserName)" got "Call function missing value"

'<input type="text" placeholder="Enter your username" id="logonAccount" name="logonAccount" class="sc-iktFfs bkQKJy sc-jQbJRc guveaB" value> == $0
CD.FindElementById("logonAccount").Click
CD.SendKeys (UserName)

'<input placeholder="Enter your password" id="logonPassword" name="logonPassword" class="sc-iktFfs bkQKJy sc-jUEmfL fsdOLa" type="password"> == $0
CD.FindElementById("logonPassword").Click
CD.SendKeys (Password)

'<button label="Login" type="submit" id="loginButton" class="sc-iqHYmW imzyTg sc-kIeSZW uCDoN"><span class="sc-gKseQn dhQxyI">Login</span></button>
CD.FindElementById("loginButton").Click

Debug.Print "Logged in to CMC"

*** END OF MY CODE AT THIS POINT, WE HAVE LOGGED IN ******************************************

'open the watchlist page:
CD.get DownloadURL, False 'so far, so good, we have MarketCap1 page open, wanting to press the Download CSV button
'
'Code from Peter_SSs
'Wait & click the button:
For I = 1 To 20
CD.Wait 500
Set tObj = CD.FindElementsByCss("input[class='inputbutton long btn']")
If tObj.Count > 0 Then Exit For
Next I
Debug.Print "i=" & I, "Count=" & tObj.Count
If I < 20 Then
dRequest = Now
CD.Wait 1000
tObj(1).Click
Debug.Print "Clicked"
Else
MsgBox ("Button not catched")
Debug.Print "Button not catched"
Stop '<<< For Debug
Exit Sub
End If
'
'Look into the destination path:
For I = 1 To 20 'Max 10 secs
CD.Wait 500
myFile = Dir(dPath & "*.csv")
Do
If myFile = "" Then Exit Do
fDnT = FileDateTime(dPath & myFile)
If fDnT > dRequest Then
fOk = True 'eg download executed
End If
If fOk Then Exit Do
myFile = Dir
Loop
If fOk Then Exit For
Next I
If fOk Then
Debug.Print "Downloaded", I, dPath & myFile
'MsgBox ("Downloaded: " & dPath & myFile)
Else
Debug.Print "Download failure", I
MsgBox ("No file has been downloaded ")
End If