53 lines
1.7 KiB
Python
53 lines
1.7 KiB
Python
import os
|
|
import sqlite3
|
|
|
|
from cfg import config
|
|
if(not config["use_sqlite"]):
|
|
import pymysql
|
|
|
|
from proxy import fetch_proxies
|
|
|
|
def get_cache(directory, name):
|
|
if(config["use_sqlite"]):
|
|
cache_file = os.path.join(directory, "{}.sqlite".format(name))
|
|
if(not os.path.exists(cache_file)):
|
|
with open(cache_file, "w") as fin:
|
|
pass
|
|
db = sqlite3.connect(cache_file)
|
|
|
|
cursor = db.cursor()
|
|
|
|
cursor.execute("CREATE TABLE proxies(proxy TEXT, lasttime_could_not_be_used DECIMAL)")
|
|
cursor.execute("CREATE TABLE links(source TEXT, destination TEXT)")
|
|
cursor.execute("CREATE TABLE dijkstra_helper(name TEXT UNIQUE, value INT)")
|
|
cursor.execute("CREATE TABLE failed_to_fetch(title TEXT, depth INT)")
|
|
|
|
db.commit()
|
|
db = sqlite3.connect(cache_file)
|
|
fetch_proxies(db)
|
|
return db
|
|
db = pymysql.connect(
|
|
host=config["mysql_server"]
|
|
, user=config["mysql_user"]
|
|
, password=config["mysql_password"]
|
|
, db=config["mysql_database"]
|
|
, charset="utf8mb4")
|
|
|
|
cursor = db.cursor()
|
|
|
|
cursor.execute("CREATE TABLE IF NOT EXISTS proxies(proxy varchar(100), lasttime_could_not_be_used DECIMAL)")
|
|
cursor.execute("CREATE TABLE IF NOT EXISTS links(source varchar(100) CHARACTER SET utf8mb4, destination varchar(100) CHARACTER SET utf8mb4)")
|
|
cursor.execute("CREATE TABLE IF NOT EXISTS dijkstra_helper(name varchar(100) CHARACTER SET utf8mb4 UNIQUE, value INT)")
|
|
cursor.execute("CREATE TABLE IF NOT EXISTS failed_to_fetch(title varchar(100) CHARACTER SET utf8mb4, depth INT)")
|
|
|
|
db.commit()
|
|
fetch_proxies(db)
|
|
return db
|
|
|
|
|
|
def clear_cache_data(connection):
|
|
cursor = connection.cursor()
|
|
cursor.execute("DELETE FROM links")
|
|
cursor.execute("DELETE FROM dijkstra_helper")
|
|
connection.commit()
|