Pulling data off the web

[Sharid]

I'm trying to pull of some urls for products of ebay, about 5 pages worth. but I cant get the code to work

In sheet2 A1 you place the keyword like "Laptops" and then IE opens goes to ebay searches Laptops and the urls are to be pulled off then it clicks on the next page tab. However I cant get it to work keeps giving me an error message

Private Sub CommandButton1_Click()
    Dim ie As Object
    Dim htmlDoc As Object
    Dim nextPageElement As Object
    Dim div As Object
    Dim link As Object
    Dim URL As String
    Dim pageNumber As Long
    Dim i As Long
    
    ' Takes seach from A1 and places it into ebay
    URL = "https://www.ebay.co.uk/sch/" & Replace(Worksheets("Sheet2").Range("A1").Value, " ", "+")

    Set ie = CreateObject("InternetExplorer.Application")
    
    With ie
        .Visible = True
        .navigate URL
        Do While .Busy Or .readyState <> 4
            DoEvents
        Loop
    End With

    Application.Wait Now + TimeSerial(0, 0, 5)
    
    Set htmlDoc = ie.document
    pageNumber = 1
    i = 2
    Do
        For Each div In htmlDoc.getElementsByTagName("div")
            If div.getAttribute("class") = "vip" Then
                Set link = div.getElementsByTagName("a")(0)
                Cells(i, 1).Value = link.getAttribute("href")
                i = i + 1
            End If
        Next div
        If pageNumber >= 5 Then Exit Do 'the first 5 pages
        Set nextPageElement = htmlDoc.getElementById("gspn-next")
        If nextPageElement Is Nothing Then Exit Do
        
        ' Clicks web next page
        nextPageElement.Click 'next web page
        Do While ie.Busy Or ie.readyState <> 4
            DoEvents
        Loop
        Application.Wait Now + TimeSerial(0, 0, 5)
        Set htmlDoc = ie.document
        pageNumber = pageNumber + 1
    Loop
ie.Quit
    Set ie = Nothing
    Set htmlDoc = Nothing
    Set nextPageElement = Nothing
    Set div = Nothing
    Set link = Nothing

 MsgBox "All Done"
End Sub

 

[Domenic]

On which line are you getting an error?

And, what type of error are you getting?

In any case, to click the next page element, try...

htmlDoc.getElementsByClassname("gspr next")(0).click

Also, it looks like you should be looping through the anchor elements in searching for the class name, not the div elements...

        For Each link In htmlDoc.getElementsByTagName("a")
            If link.getAttribute("class") = "vip" Then
                Cells(i, 1).Value = link.getAttribute("href")
                i = i + 1
            End If
        Next link

Hope this helps!

 

[Sharid]

Domenic

Thanks for your super help, the second part of the code worked, i can fetch the URLs but I can't get it to change pages, so currently the code will pull of the same urls about 5 times as I have set it to search 5 pages. This is possible to me inputting the wrong Class I have tried several but can not work it out. I have included an image below, maybe you can make sense of it.

Also in the image I have highlighted a red box, which shows how may urls to show on ebay, currently this is set to 50, this can change to 25 or 100 or 200, is there a way to set it via a cell, input, so I can select it as I will be setting IE to False. if not I would like it hard set to 200 in the code if possible. If this is not possible, no problems as I am grateful for the help you have given

 

[Domenic]

Can you post the exact code that you're using with the changes I suggested?

 

[Sharid]

As requested, please see below

Private Sub CommandButton1_Click()
    Dim ie As Object
    Dim htmlDoc As Object
    Dim nextPageElement As Object
    Dim div As Object
    Dim link As Object
    Dim URL As String
    Dim pageNumber As Long
    Dim i As Long
    
    ' Takes Url from Sheet2 A1 seach from Keyword from B1 and places IE
     URL = Sheets("Sheet2").Range("A1").Value & Replace(Worksheets("Sheet2").Range("B1").Value, " ", "+")

    Set ie = CreateObject("InternetExplorer.Application")
    
    With ie
        .Visible = True
        .navigate URL
        Do While .Busy Or .readyState <> 4
            DoEvents
        Loop
    End With

    Application.Wait Now + TimeSerial(0, 0, 5)
    Set htmlDoc = ie.document
    pageNumber = 1
    i = 2
    Do
    
    For Each link In htmlDoc.getElementsByTagName("a")
            If link.getAttribute("class") = "vip" Then
                Cells(i, 1).Value = link.getAttribute("href")
                i = i + 1
            End If
        Next link
    
        If pageNumber >= 5 Then Exit Do 'the first 5 pages
        
        Set nextPageElement = htmlDoc.getElementsByClassName("pagn-next")(0).Click
        
        If nextPageElement Is Nothing Then Exit Do
        
        ' Clicks web next page
        'nextPageElement.Click 'next web page  ' Disabled due to error message Object does not support this property or method
        Do While ie.Busy Or ie.readyState <> 4
            DoEvents
        Loop
        Application.Wait Now + TimeSerial(0, 0, 5)
        Set htmlDoc = ie.document
        pageNumber = pageNumber + 1
    Loop
ie.Quit
    Set ie = Nothing
    Set htmlDoc = Nothing
    Set nextPageElement = Nothing
    Set div = Nothing
    Set link = Nothing

 MsgBox "All Done"
End Sub

 

[Domenic]

That's because you haven't changed the class name for your next page element as I suggested. Try...

Private Sub CommandButton1_Click()
    Dim ie As Object
    Dim htmlDoc As Object
    Dim nextPageElement As Object
    Dim div As Object
    Dim link As Object
    Dim URL As String
    Dim pageNumber As Long
    Dim i As Long
    
    ' Takes Url from Sheet2 A1 seach from Keyword from B1 and places IE
     URL = Sheets("Sheet2").Range("A1").Value & Replace(Worksheets("Sheet2").Range("B1").Value, " ", "+")

    Set ie = CreateObject("InternetExplorer.Application")
    
    With ie
        .Visible = True
        .navigate URL
        Do While .Busy Or .readyState <> 4
            DoEvents
        Loop
    End With

    Application.Wait Now + TimeSerial(0, 0, 5)
    Set htmlDoc = ie.document
    pageNumber = 1
    i = 2
    Do
    
    For Each link In htmlDoc.getElementsByTagName("a")
            If link.getAttribute("class") = "vip" Then
                Cells(i, 1).Value = link.getAttribute("href")
                i = i + 1
            End If
        Next link
    
        If pageNumber >= 5 Then Exit Do 'the first 5 pages
        
        Set nextPageElement = htmlDoc.getElementsByClassName("gspr next")(0)
        
        If nextPageElement Is Nothing Then Exit Do
        
        nextPageElement.Click 'next web page
        Do While ie.Busy Or ie.readyState <> 4
            DoEvents
        Loop
        Application.Wait Now + TimeSerial(0, 0, 5)
        Set htmlDoc = ie.document
        pageNumber = pageNumber + 1
    Loop
ie.Quit
    Set ie = Nothing
    Set htmlDoc = Nothing
    Set nextPageElement = Nothing
    Set div = Nothing
    Set link = Nothing

 MsgBox "All Done"
End Sub

 

[Sharid]

Super, your a life saver, thanks for taking the time out to have a look.

 

[Domenic]

That's great, glad I was able to help, cheers!