60 lines
1.8 KiB
Python
60 lines
1.8 KiB
Python
import os
|
|
import sqlite3
|
|
|
|
from cfg import config
|
|
if(not config["use_sqlite"]):
|
|
import pymysql
|
|
|
|
from proxy import fetch_proxies
|
|
|
|
def get_cache(directory, name):
|
|
if(config["use_sqlite"]):
|
|
cache_file = os.path.join(directory, "{}.sqlite".format(name))
|
|
if(not os.path.exists(cache_file)):
|
|
with open(cache_file, "w") as fin:
|
|
pass
|
|
db = sqlite3.connect(cache_file)
|
|
|
|
cursor = db.cursor()
|
|
|
|
cursor.execute("CREATE TABLE proxies(proxy TEXT, lasttime_could_not_be_used DECIMAL)")
|
|
cursor.execute("CREATE TABLE links(source INT, destination INT)")
|
|
cursor.execute("CREATE TABLE dijkstra_helper(page INT UNIQUE, value INT)")
|
|
cursor.execute("CREATE TABLE failed_to_fetch(page INT, depth INT)")
|
|
cursor.execute("CREATE TABLE pages(title TEXT)")
|
|
|
|
db.commit()
|
|
db = sqlite3.connect(cache_file)
|
|
fetch_proxies(db)
|
|
return db
|
|
db = pymysql.connect(
|
|
host=config["mysql_server"]
|
|
, user=config["mysql_user"]
|
|
, password=config["mysql_password"]
|
|
, db=config["mysql_database"]
|
|
, charset="utf8mb4")
|
|
|
|
cursor = db.cursor()
|
|
|
|
cursor.execute("CREATE TABLE IF NOT EXISTS proxies(proxy varchar(100), lasttime_could_not_be_used DECIMAL)")
|
|
cursor.execute("CREATE TABLE IF NOT EXISTS links(source INT, destination INT)")
|
|
cursor.execute("CREATE TABLE IF NOT EXISTS dijkstra_helper(page INT UNIQUE, value INT)")
|
|
cursor.execute("CREATE TABLE IF NOT EXISTS failed_to_fetch(page INT, depth INT)")
|
|
cursor.execute('''CREATE TABLE IF NOT EXISTS
|
|
pages(
|
|
title varchar(400) CHARACTER SET utf8mb4
|
|
, page_id INT AUTO_INCREMENT
|
|
, PRIMARY KEY(page_id)
|
|
)''')
|
|
|
|
db.commit()
|
|
fetch_proxies(db)
|
|
return db
|
|
|
|
|
|
def clear_cache_data(connection):
|
|
cursor = connection.cursor()
|
|
cursor.execute("DELETE FROM links")
|
|
cursor.execute("DELETE FROM dijkstra_helper")
|
|
connection.commit()
|