scientific-programming-exer.../exam/ex01/cache.py

60 lines
1.8 KiB
Python

import os
import sqlite3
from cfg import config
if(not config["use_sqlite"]):
import pymysql
from proxy import fetch_proxies
def get_cache(directory, name):
if(config["use_sqlite"]):
cache_file = os.path.join(directory, "{}.sqlite".format(name))
if(not os.path.exists(cache_file)):
with open(cache_file, "w") as fin:
pass
db = sqlite3.connect(cache_file)
cursor = db.cursor()
cursor.execute("CREATE TABLE proxies(proxy TEXT, lasttime_could_not_be_used DECIMAL)")
cursor.execute("CREATE TABLE links(source INT, destination INT)")
cursor.execute("CREATE TABLE dijkstra_helper(page INT UNIQUE, value INT)")
cursor.execute("CREATE TABLE failed_to_fetch(page INT, depth INT)")
cursor.execute("CREATE TABLE pages(title TEXT)")
db.commit()
db = sqlite3.connect(cache_file)
fetch_proxies(db)
return db
db = pymysql.connect(
host=config["mysql_server"]
, user=config["mysql_user"]
, password=config["mysql_password"]
, db=config["mysql_database"]
, charset="utf8mb4")
cursor = db.cursor()
cursor.execute("CREATE TABLE IF NOT EXISTS proxies(proxy varchar(100), lasttime_could_not_be_used DECIMAL)")
cursor.execute("CREATE TABLE IF NOT EXISTS links(source INT, destination INT)")
cursor.execute("CREATE TABLE IF NOT EXISTS dijkstra_helper(page INT UNIQUE, value INT)")
cursor.execute("CREATE TABLE IF NOT EXISTS failed_to_fetch(page INT, depth INT)")
cursor.execute('''CREATE TABLE IF NOT EXISTS
pages(
title varchar(400) CHARACTER SET utf8mb4
, page_id INT AUTO_INCREMENT
, PRIMARY KEY(page_id)
)''')
db.commit()
fetch_proxies(db)
return db
def clear_cache_data(connection):
cursor = connection.cursor()
cursor.execute("DELETE FROM links")
cursor.execute("DELETE FROM dijkstra_helper")
connection.commit()