from cfg import config sql_statements = { "update_proxies": {"sqlite": '''UPDATE proxies SET lasttime_could_not_be_used = ? WHERE proxy = ? ''' , "mysql": '''UPDATE proxies SET lasttime_could_not_be_used = %s WHERE proxy = %s '''} , "get_proxies": {"sqlite": '''SELECT proxy, lasttime_could_not_be_used FROM proxies ORDER BY lasttime_could_not_be_used ASC''' , "mysql": '''SELECT proxy, lasttime_could_not_be_used FROM proxies ORDER BY lasttime_could_not_be_used ASC'''} , "insert_proxy": {"sqlite": "INSERT INTO proxies VALUES(?, 0)" , "mysql": "INSERT INTO proxies VALUES(%s, 0)"} , "proxy_in_db": {"sqlite": "SELECT proxy FROM proxies WHERE proxy = ?" , "mysql": "SELECT proxy FROM proxies WHERE proxy = %s"} , "insert_link": {"sqlite": "INSERT INTO links(source, destination) VALUES(?, ?)" , "mysql": "INSERT INTO links(source, destination) VALUES(%s, %s)"} , "count_failed_to_fetch": {"sqlite": "SELECT COUNT(page) FROM failed_to_fetch" , "mysql": "SELECT COUNT(page) FROM failed_to_fetch"} , "get_failed_to_fetch": {"sqlite": '''SELECT failed_to_fetch.depth, failed_to_fetch.page FROM failed_to_fetch ''' , "mysql": '''SELECT failed_to_fetch.depth, failed_to_fetch.page FROM failed_to_fetch '''} , "delete_failed_to_fetch": {"sqlite": "DELETE FROM failed_to_fetch WHERE page=?" , "mysql": "DELETE FROM failed_to_fetch WHERE page=%s"} , "count_links_from": {"sqlite": "SELECT COUNT(source) FROM links WHERE source=?" , "mysql": "SELECT COUNT(source) FROM links WHERE source=%s"} , "insert_failed_to_fetch": {"sqlite": "INSERT INTO failed_to_fetch(page, depth) VALUES(?, ?)" , "mysql": "INSERT INTO failed_to_fetch(page, depth) VALUES(%s, %s)"} , "count_links_to": {"sqlite": "SELECT COUNT(destination) FROM links WHERE destination=?" , "mysql": "SELECT COUNT(destination) FROM links WHERE destination=%s"} , "dijkstra_backtrack_one": {"sqlite": '''SELECT links.source FROM links LEFT JOIN dijkstra_helper ON links.destination=dijkstra_helper.page WHERE links.destination=? ORDER BY dijkstra_helper.value ASC LIMIT 1''' , "mysql": '''SELECT links.source FROM links LEFT JOIN dijkstra_helper ON links.destination=dijkstra_helper.page WHERE links.destination=%s ORDER BY dijkstra_helper.value ASC LIMIT 1'''} , "get_page_id": {"sqlite": "SELECT rowid FROM pages WHERE title=?" , "mysql": "SELECT page_id FROM pages WHERE title=%s"} , "insert_page": {"sqlite": "INSERT INTO pages(title) VALUES(?)" , "mysql": "INSERT INTO pages(title) VALUES(%s)"} , "get_page_title": {"sqlite": "SELECT title FROM pages WHERE rowid=?" , "mysql": "SELECT title FROM pages WHERE page_id=%s"} , "dijkstra_insert_pages": {"sqlite": '''INSERT OR IGNORE INTO dijkstra_helper(page) SELECT rowid FROM pages ''' , "mysql": '''INSERT IGNORE INTO dijkstra_helper(page) SELECT page_id FROM pages '''} , "dijkstra_set_infinity": {"sqlite": "UPDATE dijkstra_helper SET value=1e1000" , "mysql": "UPDATE dijkstra_helper SET value=2147483647"} , "dijkstra_get_to_update": {"sqlite": '''SELECT page FROM dijkstra_helper LEFT JOIN links ON links.destination=dijkstra_helper.page WHERE links.source=? AND dijkstra_helper.value>?''' , "mysql": '''SELECT page FROM dijkstra_helper LEFT JOIN links ON links.destination=dijkstra_helper.page WHERE links.source=%s AND dijkstra_helper.value>%s'''} , "dijkstra_update": {"sqlite": '''UPDATE dijkstra_helper SET value=? WHERE page IN ( SELECT destination FROM links WHERE source=?) AND dijkstra_helper.value>?''' , "mysql": '''UPDATE dijkstra_helper SET value=%s WHERE page IN ( SELECT destination FROM links WHERE source=%s) AND dijkstra_helper.value>%s'''} , "dijkstra_set_root": {"sqlite": "UPDATE dijkstra_helper SET value=0 WHERE page=?" , "mysql": "UPDATE dijkstra_helper SET value=0 WHERE page=%s"} , "get_all_page_ids": {"sqlite": "SELECT rowid FROM pages" , "mysql": "SELECT page_id FROM pages"} , "get_links": {"sqlite": "SELECT source, destination FROM links" , "mysql": "SELECT source, destination FROM links"} , "delete_dijkstra": {"sqlite": "DELETE FROM dijkstra_helper" , "mysql": "DELETE FROM dijkstra_helper"} , "insert_dijkstra_values": {"sqlite": "INSERT INTO dijkstra_helper(page, value) VALUES(?, ?)" , "mysql": "INSERT INTO dijkstra_helper(page, value) VALUES(%s, %s)"} } statements = {name: statement[config["sql_method"]] for name, statement in sql_statements.items()}