Node.js + Selenium + PlayStation.Store (part 2 of 3)

This the second article in the series of three about subject. Previous one. Next one.

Here we will write a simple script to load game descriptions:

var mysql = require('mysql');
var webdriver = require('selenium-webdriver');
 
// CREATE TABLE game_descriptions (id int(11) AUTO_INCREMENT, url varchar(256), description text, PRIMARY key(id));
var connection = mysql.createConnection({
  host     : 'localhost',
  user     : 'tolik',
  password : 'i_love_psn',
  database : 'psn'
});

// div class="descriptionTxt" itemprop="description"...

connection.query('SELECT * FROM game_urls', function (error, results, fields) {
  if (error) throw error;
  for (var i=0,n=results.length; i<n; i++) {
    let url = results[i].url;
    console.log('Load url: ', url);    
    let browser = new webdriver.Builder().usingServer().withCapabilities({'browserName': 'chrome' }).build();
    browser.get(url);
    browser.wait(webdriver.until.elementLocated(webdriver.By.className('descriptionTxt')), n * 50 * 1000).then(function(elm) {
      console.log('Description text found!');
      elm.getAttribute("innerHTML").then(function(value){
        console.log(value);
        browser.quit();
        var params = [url, value];
        var query = connection.query('INSERT INTO game_descriptions(url,description) VALUES(?,?)', params, function(err, result) {
          console.log(err);
          console.log(result);
          if (i==(n-1)) connection.end();
        });
      });
    }); 
  }
});

As usual it is pretty straightforward – init packages, connect to the DB, load the URLs from the DB for each PS4 game we have saved earlier, load URLs one by one and wait until it renders the game description, then grab the description div and put its text in the DB. Test run:

node description.js

…and after 743 page loads Chrome or Selenium WebDriver got stuck 🙂

But don’t worry! Since we were loading records from the DB and were processing it sequentially we can re-start the process simply updating the app line:

for (var i=744,n=results.length; i<n; i++) {

and we good to go again. If the SONY site crashes like “Site is not available” or “The page cannot provide the secure connection” or any other issues arise, you can simply help the browser, by refreshing the page – Selenium WebDriver is running in the background and waits for the page to load and render based on our scripted condition.

Next is How to look for PS®Move.

Опубліковано у other | Теґи: , , , , , . | Додати в закладки: постійне посилання на публікацію.

Залишити відповідь