import os import sqlite3 from cfg import config if(not config["use_sqlite"]): import pymysql from proxy import fetch_proxies def get_cache(directory, name): if(config["use_sqlite"]): cache_file = os.path.join(directory, "{}.sqlite".format(name)) if(not os.path.exists(cache_file)): with open(cache_file, "w") as fin: pass db = sqlite3.connect(cache_file) cursor = db.cursor() cursor.execute("CREATE TABLE proxies(proxy TEXT, lasttime_could_not_be_used DECIMAL)") cursor.execute("CREATE TABLE links(source TEXT, destination TEXT)") cursor.execute("CREATE TABLE dijkstra_helper(name TEXT UNIQUE, value INT)") cursor.execute("CREATE TABLE failed_to_fetch(title TEXT, depth INT)") db.commit() db = sqlite3.connect(cache_file) fetch_proxies(db) return db db = pymysql.connect( host=config["mysql_server"] , user=config["mysql_user"] , password=config["mysql_password"] , db=config["mysql_database"] , charset="utf8") cursor = db.cursor() cursor.execute("CREATE TABLE IF NOT EXISTS proxies(proxy varchar(100), lasttime_could_not_be_used DECIMAL)") cursor.execute("CREATE TABLE IF NOT EXISTS links(source varchar(50), destination varchar(50))") cursor.execute("CREATE TABLE IF NOT EXISTS dijkstra_helper(name varchar(50) UNIQUE, value INT)") cursor.execute("CREATE TABLE IF NOT EXISTS failed_to_fetch(title varchar(50), depth INT)") db.commit() return db def clear_cache_data(connection): cursor = connection.cursor() cursor.execute("DELETE FROM links") cursor.execute("DELETE FROM dijkstra_helper") connection.commit()