import os import sqlite3 from cfg import config if(not config["use_sqlite"]): import pymysql from proxy import fetch_proxies def get_cache(): if(config["use_sqlite"]): directory = config["sqlite_cache_directory"] name = config["sqlite_cache_name"] cache_file = os.path.join(directory, "{}.sqlite".format(name)) if(not os.path.exists(cache_file)): with open(cache_file, "w") as fin: pass db = sqlite3.connect(cache_file) cursor = db.cursor() cursor.execute("CREATE TABLE proxies(proxy TEXT, lasttime_could_not_be_used DECIMAL)") cursor.execute("CREATE TABLE links(source INT, destination INT)") cursor.execute("CREATE TABLE dijkstra_helper(page INT UNIQUE, value INT)") cursor.execute("CREATE TABLE failed_to_fetch(page INT, depth INT)") cursor.execute("CREATE TABLE pages(title TEXT)") db.commit() db = sqlite3.connect(cache_file) fetch_proxies(db) return db db = pymysql.connect( host=config["mysql_server"] , user=config["mysql_user"] , password=config["mysql_password"] , db=config["mysql_database"] , charset="utf8mb4") cursor = db.cursor() cursor.execute("CREATE TABLE IF NOT EXISTS proxies(proxy varchar(100), lasttime_could_not_be_used DECIMAL)") cursor.execute("CREATE TABLE IF NOT EXISTS links(source INT, destination INT)") cursor.execute("CREATE TABLE IF NOT EXISTS dijkstra_helper(page INT UNIQUE, value INT)") cursor.execute("CREATE TABLE IF NOT EXISTS failed_to_fetch(page INT, depth INT)") cursor.execute('''CREATE TABLE IF NOT EXISTS pages( title varchar(400) CHARACTER SET utf8mb4 , page_id INT AUTO_INCREMENT , PRIMARY KEY(page_id) )''') db.commit() fetch_proxies(db) return db def clear_cache_data(connection): cursor = connection.cursor() cursor.execute("DELETE FROM links") cursor.execute("DELETE FROM dijkstra_helper") connection.commit()