-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathindex.js
79 lines (61 loc) · 2.41 KB
/
index.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
var request = require('request');
var cheerio = require('cheerio');
var fs = require('fs');
var mongoose = require('mongoose');
var promise = require('promise');
var url = "http://www.d20pfsrd.com/magic/spell-lists-and-domains/spell-lists-sorcerer-and-wizard/#TOC-8th-Level-Sorcerer-Wizard-Spells";
function SpellsCrawling() {
var counter = 0;
mongoose.connect('mongodb://localhost/tp1_database');
var db = mongoose.connection;
db.on('error', console.error.bind(console, 'connection error: '));
db.once('open', function () {
console.log('Connection successful');
});
console.log('Crawling the page: ' + url);
request(url, (error, response, html) => {
if (!error && response.statusCode == 200) {
console.log('OKKKKK');
const $ = cheerio.load(html);
const toJSON = {
spells: []
}
$('.article-content table').each((i, element) => {
const level = $(element)
.children('caption')
.text();
$(element).children('tbody').each((i, element) => {
$(element).children('tr').each((i, element) => {
const spell = $(element)
.children('td')
.children('a')
.first()
.text();
toJSON.spells.push(
{
name: spell,
level: level,
components: ["V", "S", "M"],
spell_resistance: false
}
);
counter++;
console.log('Name ' + spell + ' ' + counter);
db.collection('spells').insertOne({
name: spell,
level: level,
components: ["V", "S", "M"],
spell_resistance: false
})
})
})
})
// let jsonSpells = JSON.stringify(toJSON);
// fs.writeFileSync('spells.json', jsonSpells);
}
else {
console.log('Code: ' + response.statusCode + ' Error: ' + error);
}
})
}
SpellsCrawling();