zz_deprecated:infojobs_spider
This is an old revision of the document!
- jobs_mailer.sh
#!/bin/bash TMPFILE="/tmp/jobs_today_$(date +%s).html" SENDFILE="/tmp/jobs_save_today_$(date +%s).html" OFFERMARK="table_results_offer" INFOTAGS="prefijoPuesto prefijoPoblacion prefijoEmpresa" #AVOIDTHIS="Sant Cugat Del Vall|Sabadell" AVOIDTHIS="monguiland" get_data() { #wget -O- -q $SEARCHURL | grep -A 6 "<div class=\"cell-text\">$(date +%d/%m)</div>" > $TMPFILE local AUXOFFERID="" local AUXCOMPANY="" local AUXDESC="" local AUXURL="" wget -O- -q $SEARCHURL | grep -A 15 "<div class=\"cell-text\">$(date +%d/%m)</div>" | egrep "${OFFERMARK}" | awk -F\' '{print $4}' | sed "s,',,g" | head -100 | while read AUXOFFERID do if [ $(sqlite3 $DBFILE "SELECT count(OFFERID) from offers where OFFERID like '$AUXOFFERID' ;") -eq 0 ] then sqlite3 $DBFILE "INSERT INTO offers (id, OFFERID, is_sent) VALUES(NULL, '$AUXOFFERID', 1) ;" fi done } createdb() { if ! [ -f $DBFILE ] then sqlite3 $DBFILE "create table offers (id INTEGER PRIMARY KEY, OFFERID text, is_sent boolean) ;" fi } mailer() { local let UNSENTJOBS=$(sqlite3 $DBFILE "SELECT count(id) from offers where is_sent=1 ;") local SENDME="" if [ ${UNSENTJOBS} -gt 10 ] then > $SENDFILE sqlite3 $DBFILE "SELECT OFFERID from offers where is_sent=1 ;" | while read AUXOFFERID ; do echo "http://www.infojobs.net/barcelona/linux/of-i${AUXOFFERID} " >> $SENDFILE #wget -O- -q "http://www.infojobs.net/barcelona/linux/of-i${AUXOFFERID}" | egrep "${INFOTAGS// /|}" | awk 'BEGIN{FS = "[<>\"]"} {print $9}' >> $SENDFILE SENDME=$(wget -O- -q "http://www.infojobs.net/barcelona/linux/of-i${AUXOFFERID}" | egrep "${INFOTAGS// /|}" | awk 'BEGIN{FS = "[<>\"]"} {print $9}') if [ ! "$(echo "${SENDME}" | egrep "${AVOIDTHIS}")" ]; then echo "${SENDME}" >> $SENDFILE else sqlite3 $DBFILE "UPDATE offers set is_sent=0 where OFFERID LIKE '${AUXOFFERID}' ;" fi done sqlite3 $DBFILE "UPDATE offers set is_sent=0 where is_sent=1 ;" cat $SENDFILE | mail -s "Jobs $(date +%Y/%m/%d_%H:%M)" $MAIL fi } clean_all() { for i in $SENDFILE $TMPFILE do if [ -f $i ] then rm -f $i fi done } for i in $(dirname $0)/*.conf #for i in $(dirname $0)/jholgado-v3.conf do . $i createdb get_data mailer clean_all done
zz_deprecated/infojobs_spider.1330008539.txt.gz ยท Last modified: 2012/02/23 14:49 (external edit)