scientific-programming-exercises/exam/ex01/cache.py

import os
import sqlite3

from cfg import config
if(not config["use_sqlite"]):
	import pymysql

from proxy import fetch_proxies

def get_cache(directory, name):
	if(config["use_sqlite"]):
		cache_file = os.path.join(directory, "{}.sqlite".format(name))
		if(not os.path.exists(cache_file)):
			with open(cache_file, "w") as fin:
				pass
			db = sqlite3.connect(cache_file)

			cursor = db.cursor()

			cursor.execute("CREATE TABLE proxies(proxy TEXT, lasttime_could_not_be_used DECIMAL)")
			cursor.execute("CREATE TABLE links(source INT, destination INT)")
			cursor.execute("CREATE TABLE dijkstra_helper(page INT UNIQUE, value INT)")
			cursor.execute("CREATE TABLE failed_to_fetch(page INT, depth INT)")
			cursor.execute("CREATE TABLE pages(title TEXT)")

			db.commit()
		db = sqlite3.connect(cache_file)
		fetch_proxies(db)
		return db
	db = pymysql.connect(
			host=config["mysql_server"]
			, user=config["mysql_user"]
			, password=config["mysql_password"]
			, db=config["mysql_database"]
			, charset="utf8mb4")

	cursor = db.cursor()

	cursor.execute("CREATE TABLE IF NOT EXISTS proxies(proxy varchar(100), lasttime_could_not_be_used DECIMAL)")
	cursor.execute("CREATE TABLE IF NOT EXISTS links(source INT, destination INT)")
	cursor.execute("CREATE TABLE IF NOT EXISTS dijkstra_helper(page INT UNIQUE, value INT)")
	cursor.execute("CREATE TABLE IF NOT EXISTS failed_to_fetch(page INT, depth INT)")
	cursor.execute('''CREATE TABLE IF NOT EXISTS 
			pages(
				title varchar(400) CHARACTER SET utf8mb4
				, page_id INT AUTO_INCREMENT
				, PRIMARY KEY(page_id)
				)''')

	db.commit()
	fetch_proxies(db)
	return db
	

def clear_cache_data(connection):
	cursor = connection.cursor()
	cursor.execute("DELETE FROM links")
	cursor.execute("DELETE FROM dijkstra_helper")
	connection.commit()