February 2012

NotesBridge

Grab jobs from Freelancer

This task use java and jsoup to grab jobs from Freelancer

Grab jobs from Freelancer

Create Freelancer class as following
Call Freelancer.grab() method as following

Call Freelancer.grab() method

List<Freelancer.Job> jobs = Freelancer.grab();
for (int i = 0; i < jobs.size(); i++) {
    logger.info(jobs.get(i).toString());
}

Freelancer class

public class Freelancer {

    private static Logger logger = Logger.getLogger(Freelancer.class);
     
    public static List<Job> grab() {
        List<Job> jobs = new ArrayList<Job>();
        try {
            String link = "http://api.freelancer.com/Project/Search.xml?order=submitdate&order_dir=desc";
            Document doc = Jsoup.parse(new URL(link), 60000);
            Elements elements =  doc.select("item");
            Element element = null;
            for (int i = 0; i < elements.size(); i++) {
                element = elements.get(i);
                Job job = new Job();
                if (element.select("id").first() == null) continue;
                job.id = element.select("id").first().text();
                job.url = element.select("url").first().text();
                job.title = element.select("name").first().text();
                job.shortDesc = element.select("short_descr").first().text();
                job.data.put("Start date:", element.select("start_date").first().text());
                job.data.put("End date:", element.select("end_date").first().text());
                job.data.put("Buyer url:", element.select("buyer url").first().text());
                job.data.put("Buyer id:", element.select("buyer id").first().text());
                job.data.put("Buyer username:", element.select("buyer username").first().text());
                job.data.put("Featured:", element.select("options featured").first().text());
                job.data.put("Non-public:", element.select("options nonpublic").first().text());
                job.data.put("Trial:", element.select("options trial").first().text());
                job.data.put("For gold members:", element.select("options for_gold_members").first().text());
                job.data.put("Hidden bids:", element.select("options hidden_bids").first().text());
                job.data.put("Min budget:", element.select("budget min").first().text());
                job.data.put("Max budget:", element.select("budget max").first().text());
                Elements eles = element.select("jobs item");
                String vals = "";
                for (int j = 0; j < eles.size(); j++) {
                    if (vals.length() > 0) vals += ", ";
                    vals += eles.get(j).text();
                }
                job.data.put("Jobs:", vals);
                jobs.add(job);
            }
        } catch (Exception e) {
            logger.error("", e);
        }
        return jobs;
    }
        
    public static class Job {
        public String id = "";
        public String title = "";
        public String shortDesc = "";
        public String url = "";
        public Map<String, String> data = new HashMap<String, String>();
         
        public String toString() {
            String tag = "\r\n";
            tag += "Id: " + id + "\r\n";
            tag += "Title: " + title + "\r\n";
            tag += "Url: " + url + "\r\n";
            tag += "Short Desc: " + shortDesc + "\r\n";
            for (String key : data.keySet()) {
                tag += key + " " + data.get(key) + "\r\n";
            }
            return tag;
        }
    }
     
}

Java jsoup

NotesBridge

Grab jobs from vWorker

This task use java and jsoup to grab jobs from vWorker

Grab jobs from vWorker

Create vWorker class as following
Call vWorker.grab() method as following

Call vWorker.grab() method

List<vWorker.Job> jobs = vWorker.grab(1);
for (int i = 0; i < jobs.size(); i++) {
    vWorker.Job job = jobs.get(i);
    logger.info(job.toString());
}

vWorker class

public class vWorker {

    private static Logger logger = Logger.getLogger(vWorker.class);
     
    public static List<Job> grab(int maxpage) {
        List<Job> jobs = new ArrayList<Job>();
        try {
            boolean stop = false;
            int pageno = 0;
            String link = "http://www.vworker.com/RentACoder/DotNet/misc/BidRequests/ShowBidRequests.aspx?lngBidRequestListType=3&optSortTitle=2&lngBidRequestCategoryId=-1&txtMaxNumberOfEntriesPerPage=10&optBidRequestPhase=2&lngSortColumn=-6&blnModeVerbose=True&optBiddingExpiration=1&intTabSelectedId=2";
            while (!stop) {
                Document doc = Jsoup.parse(new URL(link), 60000);
                Elements elements =  null;
                Element element = null;
              
                elements = doc.select("a");
                for (int i = 0; i < elements.size(); i++) {
                    element = elements.get(i);
                    String val = element.attr("href");
                    String pattern = "/RentACoder/misc/BidRequests/ShowBidRequest.asp?lngBidRequestId=";
                    int pos = val.indexOf(pattern);
                    if (pos < 0) continue;
                    String code = val.substring(pos + pattern.length());
                    String title = element.text();
                    if (!element.parent().tagName().equals("font")) continue;
                    if (!element.parent().parent().tagName().equals("td")) continue;
                    Node parent = element.parent().parent().parent().nextSibling().nextSibling();
                    Document pdoc = Jsoup.parse(parent.outerHtml());
                    String shortDesc = pdoc.text();
                    Job job = new Job();
                    job.id = code;
                    job.title = title;
                    job.shortDesc = shortDesc;
                    jobs.add(job);
                }
              
                elements = doc.select("input[name=cmdNextPage]");
                if (elements.size() == 0) {
                    stop = true;
                } else {
                    element = elements.get(0);
                    String val = element.attr("onclick");
                    int pos = val.indexOf("action='");
                    if (pos < 0) {
                        stop = true;
                        continue;
                    }
                    val = val.substring(pos + 8);
                    pos = val.lastIndexOf("'");
                    if (pos < 0) {
                        stop = true;
                        continue;
                    }
                    val = val.substring(0, pos);
                    link = val;
                    pageno++;
                    if (pageno >= maxpage) {
                        stop = true;
                    }
                }
            }
          
            for (int i = 0; i < jobs.size(); i++) {
                Job job = jobs.get(i);
                link = "http://vworker.com/RentACoder/misc/BidRequests/ShowBidRequest.asp?lngBidRequestId=" + job.id + "&intProjectTab_TabId=1";
                Document doc = Jsoup.parse(new URL(link), 60000);
                Elements elements =  null;
                Element element = null;
                element = doc.select("#idTabstripContent").first();
                if (element != null) {
                    elements = element.select("td");
                    String[] fields = new String[] { "Non-action ratio:", "Employer security verifications:", "Approved on:", "Bidding closes:", "Viewed (by workers):", "Deadline:", "Phase:", "Sourcing type:", "Payment Model:", "Max Accepted Bid:", "Expert Guarantee:", "Estimated size:", "Bidding type:", "Accepted bidder economy type(s):", "Accepted english fluency(ies):", "ExpertRating requirement:", "Project management:" };
                    List<String> fieldList = new ArrayList<String>();
                    for (int j = 0; j < fields.length; j++) {
                        fieldList.add(fields[j]);
                    }
                    for (int j = 0; j < elements.size(); j++) {
                        String field = elements.get(j).text();
                        String value = "";
                        if (j + 1 < elements.size()) {
                            value = elements.get(j + 1).text();
                        }
                        if (fieldList.indexOf(field) >= 0) {
                            job.data.put(field, value);
                        }
                    }
                    elements = doc.select("b");
                    for (int j = 0; j < elements.size(); j++) {
                        element = elements.get(j);
                        String name = element.text().trim();
                        if ("Brief summary:".equals(name)) {
                            element = element.nextElementSibling().nextElementSibling();
                            if (element.html().indexOf("<a name=\"NoWorkInAdvance\">") >= 0) {
                                element = element.nextElementSibling();
                            }
                            job.longDesc = element.html();
                        }
                    }
                }
            }
        } catch (Exception e) {
            logger.error("", e);
        }
        return jobs;
    }
     
    public static class Job {
        public String id = "";
        public String title = "";
        public String shortDesc = "";
        public String longDesc = "";
        public Map<String, String> data = new HashMap<String, String>();
         
        public String toString() {
            String tag = "\r\n";
            tag += "Id: " + id + "\r\n";
            tag += "Title: " + title + "\r\n";
            tag += "Short Desc: " + shortDesc + "\r\n";
            tag += "Long Desc: " + longDesc + "\r\n";
            for (String key : data.keySet()) {
                tag += key + " " + data.get(key) + "\r\n";
            }
            return tag;
        }
    }
}

Java jsoup

NotesBridge

Extract summary of web page

This task use java and jsoup to extract summary of web page.

Extract summary of web page

Create Summary class as following
Call Summary.extract() method as following

Call Summary.extract() method

String link = "http://forums.digitalpoint.com";
Document doc = Jsoup.parse(new URL(link), 60000);
String sum = Summary.extract(doc);
logger.info(sum);

Summary class

public static class Summary {
     
    public static String extract(Document doc) {
        String tag = "";
        List<String> lines = new ArrayList<String>();
         
        for (int i = 0; i < doc.children().size(); i++) {
            summary(doc.child(i), lines);
        }

        for (int i = 0; i < lines.size(); i++) {
            if (tag.length() > 0) tag += "\r\n";
            tag += lines.get(i);
        }
         
        return tag;
    }
        
    private static void summary(Element ele, List<String> lines) {
        if (ele.children().size() == 0 || allowedChildren(ele)) {
            String[] tags = new String[] { "div", "p" };
            for (int i = 0; i < tags.length; i++) {
                if (ele.tagName().equalsIgnoreCase(tags[i])) {
                    String text = ele.text().trim();
                    if (text.length() > 0) {
                        if (text.endsWith(".") || text.endsWith("?")) {
                            lines.add(text);
                        }
                    }
                }
            }
        } else {
            for (int i = 0; i < ele.children().size(); i++) {
                summary(ele.child(i), lines);
            }
        }
    }
        
    private static boolean allowedChildren(Element ele) {
        String[] tags = new String[] { "a", "b", "i", "strong" };
        for (int i = 0; i < ele.children().size(); i++) {
            Element child = ele.child(i);
            boolean found = false;
            for (int j = 0; j < tags.length; j++) {
                if (tags[j].equalsIgnoreCase(child.tagName())) {
                    found = true;
                    break;
                }
            }
            if (!found) return false;
        }
        return true;
    }
     
}

Java jsoup

NotesBridge

NotesBridge

Parse simple math expression

This task use java to parse simple math expression.

Parse simple math expression

Create Formula class as following
Call Formula.calculate() method as following

Call Formula.calculate() method

Map<String, Object> data = new HashMap<String, Object>();
data.put("a", 2);
data.put("b", 3);
String formula = "a - b * a";
Object result = Formula.calculate(data, formula);
if (result != null) {
    logger.info(result.toString());
} else {
    logger.info("NULL");
}

Formula class

public static class Formula {

    private Map<String, Object> data;
    private String formula;

    public Formula(Map<String, Object> data, String formula) {
        this.data = data;
        this.formula = formula;
    }
     
    public static Object calculate(Map<String, Object> data, String formula) {
        return new Formula(data, formula).calculate();
    }
     
    public Object calculate() {
        int pos = lastOperator(formula, new String[] { "+", "-" });
        if (pos < 0) {
            pos = lastOperator(formula, new String[] { "*", "/" });
            if (pos < 0) {
                return calculate(formula);
            } else {
                return calculate(formula.substring(0, pos).trim(), formula.charAt(pos) + "", formula.substring(pos + 1).trim());
            }
        } else {
            return calculate(formula.substring(0, pos).trim(), formula.charAt(pos) + "", formula.substring(pos + 1).trim());
        }
    }
     
    private Object calculate(String leftSide, String operator, String rightSide) {
        Object valLeft = null;
        Object valRight = null;
        int pos = lastOperator(leftSide, new String[] { "+", "-" });
        if (pos < 0) {
            pos = lastOperator(leftSide, new String[] { "*", "/" });
            if (pos < 0) {
                valLeft = calculate(leftSide);
            } else {
                valLeft = calculate(leftSide.substring(0, pos).trim(), leftSide.charAt(pos) + "", leftSide.substring(pos + 1).trim());
            }
        } else {
            valLeft = calculate(leftSide.substring(0, pos).trim(), leftSide.charAt(pos) + "", leftSide.substring(pos + 1).trim());
        }
        pos = lastOperator(rightSide, new String[] { "*", "/" });
        if (pos < 0) {
            valRight = calculate(rightSide);
        } else {
            valRight = calculate(rightSide.substring(0, pos).trim(), rightSide.charAt(pos) + "", rightSide.substring(pos + 1).trim());
        }
        return calculate(valLeft, operator, valRight);
    }
     
    private Object calculate(Object leftSide, String operator, Object rightSide) {
        Object tag = null;
        if (leftSide == null || rightSide == null) return null;
        if (isNumber(leftSide) && isNumber(rightSide)) {
            double valLeft = parseNumber(leftSide);
            double valRight = parseNumber(rightSide);
            tag = calculate(valLeft, operator, valRight);
        } else if (isString(leftSide) || isString(rightSide)) {
            if ("+".equals(operator)) {
                tag = leftSide.toString() + rightSide.toString();
            }
        }
        return tag;
    }
     
    private Object calculate(double leftSide, String operator, double rightSide) {
        if ("+".equals(operator)) {
            return leftSide + rightSide;
        } else if ("-".equals(operator)) {
            return leftSide - rightSide;
        } else if ("*".equals(operator)) {
            return leftSide * rightSide;
        } else if ("/".equals(operator) && rightSide != 0) {
            return leftSide / rightSide;
        } else {
            return null;
        }
    }
     
    private double parseNumber(Object src) {
        double tag = 0;
        try {
            tag = Double.parseDouble(src.toString());
        } catch (Exception e) {
            tag = 0;
        }
        return tag;
    }
     
    private boolean isString(Object src) {
        String kind = src.getClass().getName();
        if ("java.lang.String".equals(kind)) return true;
        return false;
    }
     
    private boolean isNumber(Object src) {
        if (src == null) return false;
        String[] types = new String[] { "java.lang.Character", "java.lang.Byte", "java.lang.Short", "java.lang.Integer", "java.lang.Long", "java.lang.Float", "java.lang.Double" };
        String kind = src.getClass().getName();
        for (int i = 0; i < types.length; i++) {
            if (types[i].equalsIgnoreCase(kind)) return true;
        }
        return false;
    }
     
    private Object calculate(String src) {
        if (data.containsKey(src)) {
            return data.get(src);
        } else {
            return null;
        }
    }
     
    private int lastOperator(String src, String[] operators) {
        int tag = -1;
        for (int i = 0; i < operators.length; i++) {
            int pos = src.lastIndexOf(operators[i]);
            if (pos < 0) continue;
            if (tag < 0) {
                tag = pos;
            } else {
                if (pos > tag) {
                    tag = pos;
                }
            }
        }
        return tag;
    }
     
}

Java

Monday, 20 February 2012

Grab jobs from Freelancer

Grab jobs from vWorker

Friday, 17 February 2012

Extract summary of web page

Task tagged by [jsoup]

Parse simple math expression

Blog Archive