100 lines
5.4 KiB
Python
100 lines
5.4 KiB
Python
|
from cfg import config
|
||
|
|
|
||
|
|
sql_statements = {
|
||
|
|
"update_proxies": {"sqlite": '''UPDATE proxies SET lasttime_could_not_be_used = ? WHERE proxy = ? '''
|
||
|
|
, "mysql": '''UPDATE proxies SET lasttime_could_not_be_used = %s WHERE proxy = %s '''}
|
||
|
|
, "get_proxies": {"sqlite": '''SELECT proxy, lasttime_could_not_be_used FROM proxies ORDER BY lasttime_could_not_be_used ASC'''
|
||
|
|
, "mysql": '''SELECT proxy, lasttime_could_not_be_used FROM proxies ORDER BY lasttime_could_not_be_used ASC'''}
|
||
|
|
, "insert_proxy": {"sqlite": "INSERT INTO proxies VALUES(?, 0)"
|
||
|
|
, "mysql": "INSERT INTO proxies VALUES(%s, 0)"}
|
||
|
|
, "proxy_in_db": {"sqlite": "SELECT proxy FROM proxies WHERE proxy = ?"
|
||
|
|
, "mysql": "SELECT proxy FROM proxies WHERE proxy = %s"}
|
||
|
|
, "insert_link": {"sqlite": "INSERT INTO links(source, destination) VALUES(?, ?)"
|
||
|
|
, "mysql": "INSERT INTO links(source, destination) VALUES(%s, %s)"}
|
||
|
|
, "count_failed_to_fetch": {"sqlite": "SELECT COUNT(page) FROM failed_to_fetch"
|
||
|
|
, "mysql": "SELECT COUNT(page) FROM failed_to_fetch"}
|
||
|
|
, "get_failed_to_fetch": {"sqlite": '''SELECT failed_to_fetch.depth, failed_to_fetch.page
|
||
|
|
FROM failed_to_fetch
|
||
|
|
'''
|
||
|
|
, "mysql": '''SELECT failed_to_fetch.depth, failed_to_fetch.page
|
||
|
|
FROM failed_to_fetch
|
||
|
|
'''}
|
||
|
|
, "delete_failed_to_fetch": {"sqlite": "DELETE FROM failed_to_fetch WHERE page=?"
|
||
|
|
, "mysql": "DELETE FROM failed_to_fetch WHERE page=%s"}
|
||
|
|
, "count_links_from": {"sqlite": "SELECT COUNT(source) FROM links WHERE source=?"
|
||
|
|
, "mysql": "SELECT COUNT(source) FROM links WHERE source=%s"}
|
||
|
|
, "insert_failed_to_fetch": {"sqlite": "INSERT INTO failed_to_fetch(page, depth) VALUES(?, ?)"
|
||
|
|
, "mysql": "INSERT INTO failed_to_fetch(page, depth) VALUES(%s, %s)"}
|
||
|
|
, "count_links_to": {"sqlite": "SELECT COUNT(destination) FROM links WHERE destination=?"
|
||
|
|
, "mysql": "SELECT COUNT(destination) FROM links WHERE destination=%s"}
|
||
|
|
, "dijkstra_backtrack_one": {"sqlite": '''SELECT links.source
|
||
|
|
FROM links
|
||
|
|
LEFT JOIN dijkstra_helper ON links.destination=dijkstra_helper.page
|
||
|
|
WHERE links.destination=?
|
||
|
|
ORDER BY dijkstra_helper.value ASC
|
||
|
|
LIMIT 1'''
|
||
|
|
, "mysql": '''SELECT links.source
|
||
|
|
FROM links
|
||
|
|
LEFT JOIN dijkstra_helper ON links.destination=dijkstra_helper.page
|
||
|
|
WHERE links.destination=%s
|
||
|
|
ORDER BY dijkstra_helper.value ASC
|
||
|
|
LIMIT 1'''}
|
||
|
|
, "get_page_id": {"sqlite": "SELECT rowid FROM pages WHERE title=?"
|
||
|
|
, "mysql": "SELECT page_id FROM pages WHERE title=%s"}
|
||
|
|
, "insert_page": {"sqlite": "INSERT INTO pages(title) VALUES(?)"
|
||
|
|
, "mysql": "INSERT INTO pages(title) VALUES(%s)"}
|
||
|
|
|
||
|
|
, "get_page_title": {"sqlite": "SELECT title FROM pages WHERE rowid=?"
|
||
|
|
, "mysql": "SELECT title FROM pages WHERE page_id=%s"}
|
||
|
|
, "dijkstra_insert_pages": {"sqlite": '''INSERT OR IGNORE INTO dijkstra_helper(page)
|
||
|
|
SELECT rowid FROM pages
|
||
|
|
'''
|
||
|
|
, "mysql": '''INSERT IGNORE INTO dijkstra_helper(page)
|
||
|
|
SELECT page_id FROM pages
|
||
|
|
'''}
|
||
|
|
, "dijkstra_set_infinity": {"sqlite": "UPDATE dijkstra_helper SET value=1e1000"
|
||
|
|
, "mysql": "UPDATE dijkstra_helper SET value=2147483647"}
|
||
|
|
, "dijkstra_get_to_update": {"sqlite": '''SELECT page
|
||
|
|
FROM dijkstra_helper
|
||
|
|
LEFT JOIN links ON links.destination=dijkstra_helper.page
|
||
|
|
WHERE links.source=?
|
||
|
|
AND dijkstra_helper.value>?'''
|
||
|
|
, "mysql": '''SELECT page
|
||
|
|
FROM dijkstra_helper
|
||
|
|
LEFT JOIN links ON links.destination=dijkstra_helper.page
|
||
|
|
WHERE links.source=%s
|
||
|
|
AND dijkstra_helper.value>%s'''}
|
||
|
|
, "dijkstra_update": {"sqlite": '''UPDATE dijkstra_helper
|
||
|
|
SET value=?
|
||
|
|
WHERE page IN (
|
||
|
|
SELECT destination
|
||
|
|
FROM links
|
||
|
|
WHERE source=?)
|
||
|
|
AND dijkstra_helper.value>?'''
|
||
|
|
, "mysql": '''UPDATE dijkstra_helper
|
||
|
|
SET value=%s
|
||
|
|
WHERE page IN (
|
||
|
|
SELECT destination
|
||
|
|
FROM links
|
||
|
|
WHERE source=%s)
|
||
|
|
AND dijkstra_helper.value>%s'''}
|
||
|
|
, "dijkstra_set_root": {"sqlite": "UPDATE dijkstra_helper SET value=0 WHERE page=?"
|
||
|
|
, "mysql": "UPDATE dijkstra_helper SET value=0 WHERE page=%s"}
|
||
|
|
, "get_all_page_ids": {"sqlite": "SELECT rowid FROM pages"
|
||
|
|
, "mysql": "SELECT page_id FROM pages"}
|
||
|
|
, "get_links": {"sqlite": "SELECT source, destination FROM links"
|
||
|
|
, "mysql": "SELECT source, destination FROM links"}
|
||
|
|
|
||
|
|
, "delete_dijkstra": {"sqlite": "DELETE FROM dijkstra_helper"
|
||
|
|
, "mysql": "DELETE FROM dijkstra_helper"}
|
||
|
|
, "insert_dijkstra_values": {"sqlite": "INSERT INTO dijkstra_helper(page, value) VALUES(?, ?)"
|
||
|
|
, "mysql": "INSERT INTO dijkstra_helper(page, value) VALUES(%s, %s)"}
|
||
|
|
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
statements = {name: statement[config["sql_method"]] for name, statement in sql_statements.items()}
|
||
|
|
|