Tuesday, 22 May 2012

Grab products from HP Shopping

Grab products from HP Shopping
This task use javascript sandbox with jsoup support to grab products from HP Shopping.
Grab products from HP Shopping
  1. Create javascript sandbox with jsoup support
  2. Create javascript as following
javascript
1function main(env, args) {
2 var catUrl = 'http://shopping.hp.com/en_US/home-office/-/products/Laptops/HP%20Pavilion';
3 var links = args.get('links');
4 var prods = grabProduct(catUrl, env);
5 for (var i = 0; i < prods.size(); i++) {
6 links.add(prods.get(i));
7 }
8}
9
10function grabProduct(catUrl, env) {
11 var tag = env.newArrayList();
12 var urls = env.newArrayList();
13 urls.add(catUrl);
14 try {
15 var conn = env.newJsoup().connect(catUrl).userAgent('Mozilla/5.001 (windows; U; NT4.0; en-US; rv:1.0) Gecko/25250101');
16 var doc = conn.timeout(60000).get();
17 var root = doc.select('.pagination-results-container').first();
18 if (root != null) {
19 var elements = root.select('a');
20 for (var i = 0; i < elements.size() - 1; i++) {
21 var element = elements.get(i);
22 if (element.hasClass('option')) continue;
23 if (element.hasClass('pngFix')) continue;
24 urls.add(element.attr('href'));
25 }
26 }
27 } catch (e) {
28 env.error(e);
29 }
30 for (var i = 0; i < urls.size(); i++) {
31 try {
32 var link = urls.get(i);
33 var conn = env.newJsoup().connect(link).userAgent('Mozilla/5.001 (windows; U; NT4.0; en-US; rv:1.0) Gecko/25250101');
34 var doc = conn.timeout(60000).get();
35 var elements = doc.select('.listing-page-bucket');
36 for (var j = 0; j < elements.size(); j++) {
37 var element = elements.get(j);
38 var children = element.select('.color-selector-img img.pngFix');
39 var image_list = '';
40 var image = '';
41 for (var k = 0; k < children.size(); k++) {
42 var child = children.get(k);
43 if (image_list.length > 0) image_list += '\n';
44 image_list += child.attr('src') + '';
45 if (child.attr('style') + '' != 'display:none') {
46 image = child.attr('src');
47 }
48 }
49 var child = element.select('.product-specs h3 a').first();
50 if (child == null) continue;
51 var title = child.text();
52 var url = child.attr('href');
53 var desc = '';
54 child = element.select('.product-specs .rating').first();
55 if (child != null) {
56 child = child.nextElementSibling().nextElementSibling();
57 desc = child.text();
58 }
59 var price = '';
60 child = element.select('#start-price').first();
61 if (child != null) {
62 price = child.text();
63 } else {
64 child = element.select('.price-value').first();
65 if (child != null) {
66 price = child.text();
67 }
68 }
69 var it = env.newHashMap();
70 it.put('image-list', image_list);
71 it.put('image', image);
72 it.put('title', title);
73 it.put('url', url);
74 it.put('desc', desc);
75 it.put('price', price);
76 tag.add(it);
77 }
78 } catch (e) {
79 env.error(e);
80 }
81 }
82 return tag;
83}
function main(env, args) {
  var catUrl = 'http://shopping.hp.com/en_US/home-office/-/products/Laptops/HP%20Pavilion';
  var links = args.get('links');
  var prods = grabProduct(catUrl, env);
  for (var i = 0; i < prods.size(); i++) {
    links.add(prods.get(i));
  }
}

function grabProduct(catUrl, env) {
  var tag = env.newArrayList();
  var urls = env.newArrayList();
  urls.add(catUrl);
  try {
      var conn = env.newJsoup().connect(catUrl).userAgent('Mozilla/5.001 (windows; U; NT4.0; en-US; rv:1.0) Gecko/25250101');
      var doc = conn.timeout(60000).get();
      var root = doc.select('.pagination-results-container').first();
      if (root != null) {
        var elements = root.select('a');
        for (var i = 0; i < elements.size() - 1; i++) {
           var element = elements.get(i);
           if (element.hasClass('option')) continue;
           if (element.hasClass('pngFix')) continue;
           urls.add(element.attr('href'));
        }
      }
  } catch (e) {
    env.error(e);
  }
  for (var i = 0; i < urls.size(); i++) {
    try {
      var link = urls.get(i);
      var conn = env.newJsoup().connect(link).userAgent('Mozilla/5.001 (windows; U; NT4.0; en-US; rv:1.0) Gecko/25250101');
      var doc = conn.timeout(60000).get();
      var elements = doc.select('.listing-page-bucket');
      for (var j = 0; j < elements.size(); j++) {
        var element = elements.get(j);
        var children = element.select('.color-selector-img img.pngFix');
        var image_list = '';
        var image = '';
        for (var k = 0; k < children.size(); k++) {
           var child = children.get(k);
           if (image_list.length > 0) image_list += '\n';
           image_list += child.attr('src') + '';
           if (child.attr('style') + '' != 'display:none') {
              image = child.attr('src');
           }
        }
        var child = element.select('.product-specs h3 a').first();
        if (child == null) continue;
        var title = child.text();
        var url = child.attr('href');
        var desc = '';
        child = element.select('.product-specs .rating').first();
        if (child != null) {
          child = child.nextElementSibling().nextElementSibling();
          desc = child.text();
        }
        var price = '';
        child = element.select('#start-price').first();
        if (child != null) {
          price = child.text();
        } else {
          child = element.select('.price-value').first();
          if (child != null) {
            price = child.text();
          }
        }
        var it = env.newHashMap();
        it.put('image-list', image_list);
        it.put('image', image);
        it.put('title', title);
        it.put('url', url);
        it.put('desc', desc);
        it.put('price', price);
        tag.add(it);
      }
    } catch (e) {
      env.error(e);
    }
  }
  return tag;
}
Java JavaScript jsoup

  Protected by Copyscape Online Copyright Protection

Posted by at 15:56 0 comments
Labels:

Grab categories from HP Shopping

Grab categories from HP Shopping
This task use javascript sandbox with jsoup support to grab categories from HP Shopping.
Grab categories from HP Shopping
  1. Create javascript sandbox with jsoup support
  2. Create javascript as following
javascript
1function main(env, args) {
2 var links = args.get('links');
3 var cats = grabCategory(env);
4 for (var i = 0; i < cats.size(); i++) {
5 links.add(cats.get(i));
6 }
7}
8
9function grabCategory(env) {
10 var tag = env.newArrayList();
11 try {
12 var link = env.newURL('http://shopping.hp.com');
13 var conn = env.newJsoup().connect(link).userAgent('Mozilla/5.001 (windows; U; NT4.0; en-US; rv:1.0) Gecko/25250101');
14 var html = conn.timeout(60000).execute().body();
15 var pat1 = 'var surveyInitData =';
16 var pat2 = "$('body').hpOnSiteExit({";
17 var pos1 = html.indexOf(pat1);
18 if (pos1 < 0) {
19 env.info('S1: javascript code not found');
20 return tag;
21 }
22 var pos2 = html.indexOf(pat2, pos1);
23 if (pos2 < 0) {
24 env.info('S2: javascript code not found');
25 return tag;
26 }
27 var js = html.substring(pos1 + pat1.length, pos2);
28 var obj = null;
29 eval('obj = ' + js);
30 var pages = obj.surveyData[0].configPages;
31 for (var i = 0; i < pages.length; i++) {
32 var pg = pages[i];
33 if (pg.pageType != 'category') continue;
34 var it = env.newHashMap();
35 it.put('title', pg.pageName);
36 it.put('url', pg.fullpath);
37 tag.add(it);
38 }
39 } catch (e) {
40 env.error(e);
41 }
42 return tag;
43}
function main(env, args) {
  var links = args.get('links');
  var cats = grabCategory(env);
  for (var i = 0; i < cats.size(); i++) {
    links.add(cats.get(i));
  }
}

function grabCategory(env) {
  var tag = env.newArrayList();
  try {
    var link = env.newURL('http://shopping.hp.com');
    var conn = env.newJsoup().connect(link).userAgent('Mozilla/5.001 (windows; U; NT4.0; en-US; rv:1.0) Gecko/25250101');
    var html = conn.timeout(60000).execute().body();
    var pat1 = 'var surveyInitData =';
    var pat2 = "$('body').hpOnSiteExit({";
    var pos1 = html.indexOf(pat1);
    if (pos1 < 0) {
      env.info('S1: javascript code not found');
      return tag;
    }
    var pos2 = html.indexOf(pat2, pos1);
    if (pos2 < 0) {
      env.info('S2: javascript code not found');
      return tag;
    }
    var js = html.substring(pos1 + pat1.length, pos2);
    var obj = null;
    eval('obj = ' + js);
    var pages = obj.surveyData[0].configPages;
    for (var i = 0; i < pages.length; i++) {
      var pg = pages[i];
      if (pg.pageType != 'category') continue;
      var it = env.newHashMap();
      it.put('title', pg.pageName);
      it.put('url', pg.fullpath);
      tag.add(it);
    }
  } catch (e) {
    env.error(e);
  }
  return tag;
}
Java JavaScript jsoup

  Protected by Copyscape Online Copyright Protection

Posted by at 15:50 0 comments
Labels:
Subscribe to: Posts (Atom)