VBA: GetElementbyClassName

jamescooper

Well-known Member
Joined
Sep 8, 2014
Messages
840
Hello,

Cannot quite work out what I am doing wrong to pull the initial price from the website listed below. It should be £585 but it is falling down on the code:

Code:
price = html.getElementsByID("_tyxjp1").innerText

Any help is greatly appreciated.

Many thanks.

VBA Code:
Sub Get_Web_Data()

Dim request As Object
Dim response As String
Dim html As New HTMLDocument
Dim website As String
Dim price As Variant

website = "https://www.airbnb.co.uk/s/Cardiff-city-centre--Cardiff/homes?adults=4&place_id=ChIJE5-LOrccbkgRzfLvM7ow5xU&checkin=2022-06-18&checkout=2022-06-19&tab_id=home_tab&refinement_paths%5B%5D=%2Fhomes&query=Cardiff%20city%20centre%2C%20Cardiff&flexible_trip_lengths%5B%5D=one_week&date_picker_type=calendar&source=structured_search_input_header&search_type=filter_change&ne_lat=51.49207564654211&ne_lng=-3.1483986320495774&sw_lat=51.468395512898894&sw_lng=-3.2042314949035813&zoom=14&search_by_map=true&room_types%5B%5D=Entire%20home%2Fapt"

Set request = CreateObject("MSXML2.XMLHTTP")

request.Open "GET", website, False

'fresh data
'request.SetRequestHeader "If-Modified-Since", "Sun, 22 May 2022 00:00:00 GMT"

request.send

response = StrConv(request.responseBody, vbUnicode)

html.body.innerHTML = response

price = html.getElementsByID("_tyxjp1").innerText

MsgBox price

End Sub
 
Sorry Anthony, it has been running great until I encountered a problem recently.

Using this in C2


On this line of code:
VBA Code:
    Set aColl = picColl(i + 1).FindElementsByTag("a")

It is giving the error message:
"Index was outside the bounds of the array"

Any ideas why this would be. Many thanks.

Code is still:

Code:
Sub ABBInfoV22()

Dim wPage As Object
''Dim wPage As Selenium.WebDriver
Dim myUrl As String, i As Long
Dim NextR As Long, NextP As Long
Dim pColl As Object, aptColl As Object
Dim LastUrl As Long, vOff As Long, eCnt As Long, bCnt As Long
Dim picColl As Object, aColl As Object, LoopMode As Boolean
Dim uSh As Worksheet, mMsg As String, uRan As Range
Dim dtToday As String

dtToday = Format(Date, "dd/mm/yyyy")
Lastrow = Sheets("Data").Range("A" & Rows.Count).End(xlUp).Row

Application.Calculation = xlManual

Set uRan = Sheets("date requests").Range("C2")             '<<< The starting position of URLs
'
'Crea Driver:
Set wPage = CreateObject("Selenium.CHRomedriver")
ReUrl:
'Loop For each Url:
'myUrl = uSh.Cells(2, "C").Offset(vOff, 0).Value
myUrl = uRan.Offset(vOff, 0).Value
If InStr(1, myUrl, "http", vbTextCompare) <> 1 Then
    AppActivate (Application.Caption)
    mMsg = "Completed, " & vOff & " Url(s), " & bCnt & " blocks, " & eCnt & " Elements"
    MsgBox (mMsg)
    Debug.Print mMsg & vbCrLf
    GoTo SQuit
End If
vOff = vOff + 1
If uRan.Offset(vOff, 0).Value <> "" Then
    LoopMode = True
End If
Debug.Print ">>>> Start, LoopMode=" & LoopMode & ", URL=" & vOff
'
wPage.Get myUrl
Sheets("data").Select
Cells(Rows.Count, 1).End(xlUp).Offset(1, 0) = myUrl

If LoopMode = False Then
    AppActivate (Application.Caption)
    MsgBox ("Now you may modify and re-execute the Search on the Chrome window" & vbCrLf _
        & "When you have done, close the MessageBox to Continue")
End If
'
'Extracting block of info:
ReLoop:
bCnt = bCnt + 1
'Wait for the list to be ready:
For i = 1 To 10
    Set aptColl = wPage.FindElementsByClass("g1tup9az")
    wPage.Wait 400
    If aptColl.Count > 0 Then Exit For
Next i
NextP = NextP + 1
'lock the collection of elements:
Set aptColl = wPage.FindElementsByClass("g1tup9az")
Set picColl = wPage.FindElementsByClass("c14whb16")
Debug.Print "Apt found=" & aptColl.Count, "I=" & i, "Page=" & NextP
Range("A1:G1").Value = Array("Description", "Overview", "Price", "Check-in date", "Check-out date", "Date checked", "Days out")
'Read each element:
For i = 1 To aptColl.Count
    eCnt = eCnt + 1
    Set aColl = picColl(i + 1).FindElementsByTag("a")
    NextR = Cells(Rows.Count, 1).End(xlUp).Row + 1
    Set pColl = aptColl(i).FindElementsByTag("div")
    Cells(NextR, 1) = pColl(1).Text
    Cells(NextR, 1).Style = "Normal"
    If aColl.Count > 0 Then                                     'Add the hyperlink
        ActiveSheet.Hyperlinks.Add Anchor:=Cells(NextR, 1), _
               Address:=aColl(1).Attribute("href")
    End If
    Cells(NextR, 2) = Replace(pColl(2).Text, Chr(10), " ", , , vbTextCompare)
    Cells(NextR, 3) = Replace(pColl(7).Text, Chr(10), " ", , , vbTextCompare)
    Cells(NextR, 4) = Mid(myUrl, 117, 10)
    Cells(NextR, 5) = Mid(myUrl, 137, 10)
    Cells(NextR, 6) = dtToday
    DoEvents
Next i
'
'Accept cookies
Set aptColl = wPage.FindElementsByClass("_148dgdpk")
If aptColl.Count = 1 Then aptColl(1).Click: wPage.Wait 100
'
'Search Next & click:
Set aptColl = wPage.FindElementsByClass("_jro6t0")
If aptColl.Count > 0 Then
    Set pColl = aptColl(1).FindElementsByTag("a")
    For i = 1 To pColl.Count
        If pColl(i).Attribute("aria-label") = "Next" Then
            pColl(i).Click
            Debug.Print "Next 20"
            wPage.Wait 990
            GoTo ReLoop
        End If
    Next i
End If
If LoopMode Then
    GoTo ReUrl
Else

    AppActivate (Application.Caption)
    mMsg = "Completed, " & "1 Url(s), " & bCnt & " blocks, " & eCnt & " Elements"
    MsgBox (mMsg)
    Debug.Print mMsg & vbCrLf
End If
Beep
SQuit:
wPage.Quit
Set wPage = Nothing

Application.Calculation = xlAutomatic

End Sub
 
Upvote 0

Excel Facts

Does the VLOOKUP table have to be sorted?
No! when you are using an exact match, the VLOOKUP table can be in any order. Best-selling items at the top is actually the best.
The macro writes some debug information in the vba "Immediate" window; so please open the Immediate windows (typing Contr-g should do the job, or Menu /View /Immediate window) and copy the information starting from the line that starts with ">>>> Start". I don't think there is confidentional info there, if you need can repèlace with * each character that you wish to hide.
If you can provide a couple of url to test that would also be great
 
Upvote 0
Dear friend, these automations are just a matter of time and patience, and they have an unknown expiration date.
I mean that tomorrow, or next week, or next month, or xxxx, the webmaster will decide to introduce a small upgrading to his webpage that will make our code to fail, either with a run-time error or just fetching the wrong data. In short, the question is not "Will it expire?" but "When will it expire?"

That said, here it is my "last penultimate" version:
VBA Code:
Sub ABBInfoV33()
Dim wPage As Object
'Dim wPage As Selenium.WebDriver

Dim myUrl As String, I As Long
Dim NextR As Long, NextP As Long
Dim pColl As Object, aptColl As Object
Dim LastUrl As Long, vOff As Long, eCnt As Long, bCnt As Long
Dim picColl As Object, aColl As Object, LoopMode As Boolean
Dim uSh As Worksheet, mMsg As String, uRan As Range
Dim aptCollTop As Object
'
'
Set uRan = Sheets("Sheet1").Range("C2")             '<<< The starting position of URLs
'
'Crea Driver:
Set wPage = CreateObject("Selenium.CHRomedriver")
ReUrl:
'Loop for each Url:
'''myUrl = uSh.Cells(2, "C").Offset(vOff, 0).Value
myUrl = uRan.Offset(vOff, 0).Value
If InStr(1, myUrl, "http", vbTextCompare) <> 1 Then
    AppActivate (Application.Caption)
    mMsg = "Completed, " & vOff & " Url(s), " & bCnt & " blocks, " & eCnt & " Elements"
    MsgBox (mMsg)
    Debug.Print mMsg & vbCrLf
    GoTo SQuit
End If
vOff = vOff + 1
If uRan.Offset(vOff, 0).Value <> "" Then
    LoopMode = True
End If
Debug.Print ">>>> Start, LoopMode=" & LoopMode & ", URL=" & vOff
'
wPage.Get myUrl
Sheets("Main").Select
Cells(Rows.Count, 1).End(xlUp).Offset(1, 0) = myUrl

If LoopMode = False Then
    AppActivate (Application.Caption)
    MsgBox ("Now you may modify and re-execute the Search on the Chrome window" & vbCrLf _
        & "When you are ready close the MessageBox to Continue")
    '
End If
'
'Extracting block of info:
ReLoop:
bCnt = bCnt + 1
'Wait for the list to be ready:
For I = 1 To 10
    Set aptColl = wPage.FindElementsByClass("g1tup9az")
    wPage.Wait 400
    If aptColl.Count > 0 Then Exit For
Next I
NextP = NextP + 1
'lock the collection of elements:
Set aptCollTop = wPage.FindElementsByClass("c12h3gv8")
''Set aptColl = wPage.FindElementsByClass("g1tup9az")
''Set picColl = wPage.FindElementsByClass("c14whb16")
Debug.Print "Apt found=" & aptCollTop.Count, "I=" & I, "Page=" & NextP
Range("A1:C1").Value = Array("Description", "Overview", "Price")
'Read each element:
For I = 1 To aptCollTop.Count
    Set aptColl = aptCollTop(I).FindElementsByClass("g1tup9az")
    Set aColl = aptCollTop(I).FindElementsByTag("a")
    eCnt = eCnt + 1
''    Set aColl = picColl(I + 1).FindElementsByTag("a")
    NextR = Cells(Rows.Count, 1).End(xlUp).Row + 1
    Set pColl = aptCollTop(I).FindElementsByClass("t1jojoys")
    Cells(NextR, 1) = pColl(1).Text
    Cells(NextR, 1).Style = "Normal"
    If aColl.Count > 0 Then                                     'Add the hyperlink
        ActiveSheet.Hyperlinks.Add Anchor:=Cells(NextR, 1), _
               Address:=aColl(1).Attribute("href")
    End If
    Cells(NextR, 2) = Replace(aptCollTop(I).FindElementsByClass("s1cjsi4j")(1).Text, Chr(10), " ", , , vbTextCompare)
    Cells(NextR, 3) = Replace(aptCollTop(I).FindElementsByClass("p1v28t5c")(1).Text, Chr(10), " ", , , vbTextCompare)
    DoEvents
Next I
'
'Accept cookies
Set aptColl = wPage.FindElementsByClass("_148dgdpk")
If aptColl.Count = 1 Then aptColl(1).Click: wPage.Wait 100
'
'Search Next & click:
Set aptColl = wPage.FindElementsByClass("_jro6t0")
If aptColl.Count > 0 Then
    Set pColl = aptColl(1).FindElementsByTag("a")
    For I = 1 To pColl.Count
        If pColl(I).Attribute("aria-label") = "Next" Then
            pColl(I).Click
            Debug.Print "Next 20"
            wPage.Wait 990
            GoTo ReLoop
        End If
    Next I
End If
If LoopMode Then
    GoTo ReUrl
Else
    AppActivate (Application.Caption)
    mMsg = "Completed, " & "1 Url(s), " & bCnt & " blocks, " & eCnt & " Elements"
    MsgBox (mMsg)
    Debug.Print mMsg & vbCrLf
End If
Beep
SQuit:
wPage.Quit
Set wPage = Nothing
End Sub
 
Upvote 0
Dear friend, these automations are just a matter of time and patience, and they have an unknown expiration date.
I mean that tomorrow, or next week, or next month, or xxxx, the webmaster will decide to introduce a small upgrading to his webpage that will make our code to fail, either with a run-time error or just fetching the wrong data. In short, the question is not "Will it expire?" but "When will it expire?"

That said, here it is my "last penultimate" version:
VBA Code:
Sub ABBInfoV33()
Dim wPage As Object
'Dim wPage As Selenium.WebDriver

Dim myUrl As String, I As Long
Dim NextR As Long, NextP As Long
Dim pColl As Object, aptColl As Object
Dim LastUrl As Long, vOff As Long, eCnt As Long, bCnt As Long
Dim picColl As Object, aColl As Object, LoopMode As Boolean
Dim uSh As Worksheet, mMsg As String, uRan As Range
Dim aptCollTop As Object
'
'
Set uRan = Sheets("Sheet1").Range("C2")             '<<< The starting position of URLs
'
'Crea Driver:
Set wPage = CreateObject("Selenium.CHRomedriver")
ReUrl:
'Loop for each Url:
'''myUrl = uSh.Cells(2, "C").Offset(vOff, 0).Value
myUrl = uRan.Offset(vOff, 0).Value
If InStr(1, myUrl, "http", vbTextCompare) <> 1 Then
    AppActivate (Application.Caption)
    mMsg = "Completed, " & vOff & " Url(s), " & bCnt & " blocks, " & eCnt & " Elements"
    MsgBox (mMsg)
    Debug.Print mMsg & vbCrLf
    GoTo SQuit
End If
vOff = vOff + 1
If uRan.Offset(vOff, 0).Value <> "" Then
    LoopMode = True
End If
Debug.Print ">>>> Start, LoopMode=" & LoopMode & ", URL=" & vOff
'
wPage.Get myUrl
Sheets("Main").Select
Cells(Rows.Count, 1).End(xlUp).Offset(1, 0) = myUrl

If LoopMode = False Then
    AppActivate (Application.Caption)
    MsgBox ("Now you may modify and re-execute the Search on the Chrome window" & vbCrLf _
        & "When you are ready close the MessageBox to Continue")
    '
End If
'
'Extracting block of info:
ReLoop:
bCnt = bCnt + 1
'Wait for the list to be ready:
For I = 1 To 10
    Set aptColl = wPage.FindElementsByClass("g1tup9az")
    wPage.Wait 400
    If aptColl.Count > 0 Then Exit For
Next I
NextP = NextP + 1
'lock the collection of elements:
Set aptCollTop = wPage.FindElementsByClass("c12h3gv8")
''Set aptColl = wPage.FindElementsByClass("g1tup9az")
''Set picColl = wPage.FindElementsByClass("c14whb16")
Debug.Print "Apt found=" & aptCollTop.Count, "I=" & I, "Page=" & NextP
Range("A1:C1").Value = Array("Description", "Overview", "Price")
'Read each element:
For I = 1 To aptCollTop.Count
    Set aptColl = aptCollTop(I).FindElementsByClass("g1tup9az")
    Set aColl = aptCollTop(I).FindElementsByTag("a")
    eCnt = eCnt + 1
''    Set aColl = picColl(I + 1).FindElementsByTag("a")
    NextR = Cells(Rows.Count, 1).End(xlUp).Row + 1
    Set pColl = aptCollTop(I).FindElementsByClass("t1jojoys")
    Cells(NextR, 1) = pColl(1).Text
    Cells(NextR, 1).Style = "Normal"
    If aColl.Count > 0 Then                                     'Add the hyperlink
        ActiveSheet.Hyperlinks.Add Anchor:=Cells(NextR, 1), _
               Address:=aColl(1).Attribute("href")
    End If
    Cells(NextR, 2) = Replace(aptCollTop(I).FindElementsByClass("s1cjsi4j")(1).Text, Chr(10), " ", , , vbTextCompare)
    Cells(NextR, 3) = Replace(aptCollTop(I).FindElementsByClass("p1v28t5c")(1).Text, Chr(10), " ", , , vbTextCompare)
    DoEvents
Next I
'
'Accept cookies
Set aptColl = wPage.FindElementsByClass("_148dgdpk")
If aptColl.Count = 1 Then aptColl(1).Click: wPage.Wait 100
'
'Search Next & click:
Set aptColl = wPage.FindElementsByClass("_jro6t0")
If aptColl.Count > 0 Then
    Set pColl = aptColl(1).FindElementsByTag("a")
    For I = 1 To pColl.Count
        If pColl(I).Attribute("aria-label") = "Next" Then
            pColl(I).Click
            Debug.Print "Next 20"
            wPage.Wait 990
            GoTo ReLoop
        End If
    Next I
End If
If LoopMode Then
    GoTo ReUrl
Else
    AppActivate (Application.Caption)
    mMsg = "Completed, " & "1 Url(s), " & bCnt & " blocks, " & eCnt & " Elements"
    MsgBox (mMsg)
    Debug.Print mMsg & vbCrLf
End If
Beep
SQuit:
wPage.Quit
Set wPage = Nothing
End Sub
Thanks for this further effort.

This one pulled the URL but not the actual data. Not sure why that was? Did it work for you?
 
Upvote 0
I succeded in demonstrating that changes occurs overnight, as this afternoon there isn't any item with classname c12h3gv8, that yesterday evening I used as the main key for retrieving the data.

So in this unclassifiable release I changed again approach:
VBA Code:
Sub ABBInfoV33()
Dim wPage As Object
'Dim wPage As Selenium.WebDriver

Dim myUrl As String, I As Long
Dim NextR As Long, NextP As Long
Dim pColl As Object, aptColl As Object
Dim LastUrl As Long, vOff As Long, eCnt As Long, bCnt As Long
Dim picColl As Object, aColl As Object, LoopMode As Boolean
Dim uSh As Worksheet, mMsg As String, uRan As Range
Dim aptCollTop As Object
'
'
Set uRan = Sheets("Sheet1").Range("C2")             '<<< The starting position of URLs
'
'Crea Driver:
Set wPage = CreateObject("Selenium.CHRomedriver")
ReUrl:
'Loop for each Url:
'''myUrl = uSh.Cells(2, "C").Offset(vOff, 0).Value
myUrl = uRan.Offset(vOff, 0).Value
If InStr(1, myUrl, "http", vbTextCompare) <> 1 Then
    AppActivate (Application.Caption)
    mMsg = "Completed, " & vOff & " Url(s), " & bCnt & " blocks, " & eCnt & " Elements"
    MsgBox (mMsg)
    Debug.Print mMsg & vbCrLf
    GoTo SQuit
End If
vOff = vOff + 1
If uRan.Offset(vOff, 0).Value <> "" Then
    LoopMode = True
End If
Debug.Print ">>>> Start, LoopMode=" & LoopMode & ", URL=" & vOff
'
wPage.Get myUrl
Sheets("Main").Select
Cells(Rows.Count, 1).End(xlUp).Offset(1, 0) = myUrl

If LoopMode = False Then
    AppActivate (Application.Caption)
    MsgBox ("Now you may modify and re-execute the Search on the Chrome window" & vbCrLf _
        & "When you are ready close the MessageBox to Continue")
    '
End If
'
'Extracting block of info:
ReLoop:
bCnt = bCnt + 1
'Wait for the list to be ready:
For I = 1 To 10
    Set aptColl = wPage.FindElementsByClass("g1tup9az")
    wPage.Wait 400
    If aptColl.Count > 0 Then Exit For
Next I
NextP = NextP + 1
'lock the collection of elements:
Set aptCollTop = wPage.FindElementsByClass("c12h3gv8")
''Set aptColl = wPage.FindElementsByClass("g1tup9az")
''Set picColl = wPage.FindElementsByClass("c14whb16")
Debug.Print "Apt found=" & aptCollTop.Count, "I=" & I, "Page=" & NextP
Range("A1:C1").Value = Array("Description", "Overview", "Price")
'Read each element:
For I = 1 To aptCollTop.Count
    Set aptColl = aptCollTop(I).FindElementsByClass("g1tup9az")
    Set aColl = aptCollTop(I).FindElementsByTag("a")
    eCnt = eCnt + 1
''    Set aColl = picColl(I + 1).FindElementsByTag("a")
    NextR = Cells(Rows.Count, 1).End(xlUp).Row + 1
    Set pColl = aptCollTop(I).FindElementsByClass("t1jojoys")
    Cells(NextR, 1) = pColl(1).Text
    Cells(NextR, 1).Style = "Normal"
    If aColl.Count > 0 Then                                     'Add the hyperlink
        ActiveSheet.Hyperlinks.Add Anchor:=Cells(NextR, 1), _
               Address:=aColl(1).Attribute("href")
    End If
    Cells(NextR, 2) = Replace(aptCollTop(I).FindElementsByClass("s1cjsi4j")(1).Text, Chr(10), " ", , , vbTextCompare)
    Cells(NextR, 3) = Replace(aptCollTop(I).FindElementsByClass("p1v28t5c")(1).Text, Chr(10), " ", , , vbTextCompare)
    DoEvents
Next I
'
'Accept cookies
Set aptColl = wPage.FindElementsByClass("_148dgdpk")
If aptColl.Count = 1 Then aptColl(1).Click: wPage.Wait 100
'
'Search Next & click:
Set aptColl = wPage.FindElementsByClass("_jro6t0")
If aptColl.Count > 0 Then
    Set pColl = aptColl(1).FindElementsByTag("a")
    For I = 1 To pColl.Count
        If pColl(I).Attribute("aria-label") = "Next" Then
            pColl(I).Click
            Debug.Print "Next 20"
            wPage.Wait 990
            GoTo ReLoop
        End If
    Next I
End If
If LoopMode Then
    GoTo ReUrl
Else
    AppActivate (Application.Caption)
    mMsg = "Completed, " & "1 Url(s), " & bCnt & " blocks, " & eCnt & " Elements"
    MsgBox (mMsg)
    Debug.Print mMsg & vbCrLf
End If
Beep
SQuit:
wPage.Quit
Set wPage = Nothing
End Sub
At a first look the information collected seem correct, go on with in-depth testing
 
Upvote 0
The two images show what I get using right now your latest link.
One pic shows the first 4 results as shown on the Chrome page; the other shows what the macro collect

I seem that the two information are equals.

If you run your test and got different results then you should details the test you run, the results you got and show the discrepancies
 

Attachments

  • CARDIFF_1_Immagine 2022-06-21 012510.jpg
    CARDIFF_1_Immagine 2022-06-21 012510.jpg
    113.5 KB · Views: 10
  • CARDIFF_3_Immagine 2022-06-21 013110.jpg
    CARDIFF_3_Immagine 2022-06-21 013110.jpg
    170.4 KB · Views: 11
Upvote 0
Could you please confirm the exact code you used to get these results because I am still receiving 0 elements.

Many thanks.
 
Upvote 0

Similar threads

Forum statistics

Threads
1,223,164
Messages
6,170,444
Members
452,326
Latest member
johnshaji

We've detected that you are using an adblocker.

We have a great community of people providing Excel help here, but the hosting costs are enormous. You can help keep this site running by allowing ads on MrExcel.com.
Allow Ads at MrExcel

Which adblocker are you using?

Disable AdBlock

Follow these easy steps to disable AdBlock

1)Click on the icon in the browser’s toolbar.
2)Click on the icon in the browser’s toolbar.
2)Click on the "Pause on this site" option.
Go back

Disable AdBlock Plus

Follow these easy steps to disable AdBlock Plus

1)Click on the icon in the browser’s toolbar.
2)Click on the toggle to disable it for "mrexcel.com".
Go back

Disable uBlock Origin

Follow these easy steps to disable uBlock Origin

1)Click on the icon in the browser’s toolbar.
2)Click on the "Power" button.
3)Click on the "Refresh" button.
Go back

Disable uBlock

Follow these easy steps to disable uBlock

1)Click on the icon in the browser’s toolbar.
2)Click on the "Power" button.
3)Click on the "Refresh" button.
Go back
Back
Top