-
Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy pathcrawler.js
More file actions
57 lines (42 loc) · 1.43 KB
/
crawler.js
File metadata and controls
57 lines (42 loc) · 1.43 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
const puppeteer = require('puppeteer');
const devices = require('puppeteer/DeviceDescriptors');
const iPhone = devices['iPhone 6'];
const fs = require('fs');
let browser;
let page;
module.exports = crawler = async () => {
browser = await puppeteer.launch({headless: false});
page = await browser.newPage();
page.setDefaultNavigationTimeout(300000);
await page.emulate(iPhone);
await page.goto("https://www.rehberlikservisi.net/category/film-kosesi/");
const findedMovie = '#orta > div.sol > div.anasayfaliste > ul > li:nth-child(1) > a > div.baslik';
await page.waitForSelector(findedMovie);
await page.click(findedMovie);
const tmpUrl = page.url();
await page.waitFor(5000);
const img = await page.evaluate(()=>{
let imgLink = document.querySelector('#orta > div.sol > div.icerikalani > p:nth-child(1) > strong > img').getAttribute('src');
return imgLink;
});
const result = await page.evaluate(() => {
let title = document.querySelector('h1').innerText;
let data = document.querySelector('#orta > div.sol > div.icerikalani').innerText;
data = data.replace(/(\r\n|\r|\n){2,}/g, '$1\n');
return {
title,
data,
imgDir: ''
}
});
var viewSource = await page.goto(img);
fs.writeFile("./images/anu.jpg", await viewSource.buffer(), function(err) {
if(err) {
return console.log(err);
}
console.log("The file was saved!");
});
result.imgDir = "./images/anu.jpg";
browser.close();
return result;
};