- Sep 22, 2019
- 111
- 33
- 28
I am having some challenges using the web scraper to get some data from upwork.com. The scraping seems to work but I can't get it to scrape more pages automatically. It seems to just scrape the first page and stops. How can I solve this? Here's my sitemap.
{"_id":"jobs","startUrl":["https://www.upwork.com/search/jobs/"],"selectors":[{"id":"Project links","type":"SelectorLink","parentSelectors":["_root"],"selector":"a.job-title-link","multiple":true,"delay":0},{"id":"project title","type":"SelectorText","parentSelectors":["_root"],"selector":"a.job-title-link","multiple":true,"regex":"","delay":0},{"id":"sing - project title","type":"SelectorText","parentSelectors":["Project links"],"selector":"header h2","multiple":false,"regex":"","delay":0},{"id":"sing-project desc","type":"SelectorText","parentSelectors":["Project links"],"selector":".job-description div","multiple":false,"regex":"","delay":0},{"id":"budget","type":"SelectorText","parentSelectors":["Project links"],"selector":".job-features li:nth-of-type(1)","multiple":false,"regex":"","delay":0},{"id":"skills","type":"SelectorText","parentSelectors":["Project links"],"selector":"section:nth-of-type(5)","multiple":false,"regex":"","delay":0},{"id":"pagination","type":"SelectorLink","parentSelectors":["_root","pagination"],"selector":".pagination li.ng-scope:nth-of-type(n+3) a","multiple":false,"delay":0}]}
{"_id":"jobs","startUrl":["https://www.upwork.com/search/jobs/"],"selectors":[{"id":"Project links","type":"SelectorLink","parentSelectors":["_root"],"selector":"a.job-title-link","multiple":true,"delay":0},{"id":"project title","type":"SelectorText","parentSelectors":["_root"],"selector":"a.job-title-link","multiple":true,"regex":"","delay":0},{"id":"sing - project title","type":"SelectorText","parentSelectors":["Project links"],"selector":"header h2","multiple":false,"regex":"","delay":0},{"id":"sing-project desc","type":"SelectorText","parentSelectors":["Project links"],"selector":".job-description div","multiple":false,"regex":"","delay":0},{"id":"budget","type":"SelectorText","parentSelectors":["Project links"],"selector":".job-features li:nth-of-type(1)","multiple":false,"regex":"","delay":0},{"id":"skills","type":"SelectorText","parentSelectors":["Project links"],"selector":"section:nth-of-type(5)","multiple":false,"regex":"","delay":0},{"id":"pagination","type":"SelectorLink","parentSelectors":["_root","pagination"],"selector":".pagination li.ng-scope:nth-of-type(n+3) a","multiple":false,"delay":0}]}