Files
m1_bdd_comparaison_perf/main.py
2024-12-22 19:17:00 +01:00

371 lines
17 KiB
Python

import random
from MonetDB import *
from PostgreSQL import *
from datetime import datetime
#Bilioteques nécessaires (pour se connecter a monetDB et postgreeSQL
#pip install psycopg2
#pip install pymonetdb
#Monet DB
requeteSelect1MonetDB = """SELECT * FROM flights WHERE "AIRLINE" = 'AA';"""
requeteSelect2MonetDB = """SELECT * FROM flights;"""
requeteSELECT3MonetDB = """SELECT * FROM flights WHERE "YEAR" = 2015 AND "MONTH" = 12 AND "DAY" = 32 AND "DAY_OF_WEEK" = 4 AND "AIRLINE" = 'B6' AND "FLIGHT_NUMBER" = 839;"""
requeteSelect05MonetDB = """SELECT * FROM flights WHERE "DAY" > 5;"""
requeteSelect10MonetDB = """SELECT * FROM flights WHERE "DAY" > 10;"""
requeteSelect15MonetDB = """SELECT * FROM flights WHERE "DAY" > 15;"""
requeteSelect20MonetDB = """SELECT * FROM flights WHERE "DAY" > 20;"""
requeteSelect25MonetDB = """SELECT * FROM flights WHERE "DAY" > 25;"""
requeteSelect30MonetDB = """SELECT * FROM flights WHERE "DAY" > 30;"""
requeteSelect05_2MonetDB = """SELECT * FROM flights WHERE "DAY" > 5 AND "DAY_OF_WEEK"<=5 AND "SCHEDULED_DEPARTURE">1200; """
requeteSelect10_2MonetDB = """SELECT * FROM flights WHERE "DAY" > 10 AND "DAY_OF_WEEK"<=5 AND "SCHEDULED_DEPARTURE">1200; """
requeteSelect15_2MonetDB = """SELECT * FROM flights WHERE "DAY" > 15 AND "DAY_OF_WEEK"<=5 AND "SCHEDULED_DEPARTURE">1200; """
requeteSelect20_2MonetDB = """SELECT * FROM flights WHERE "DAY" > 20 AND "DAY_OF_WEEK"<=5 AND "SCHEDULED_DEPARTURE">1200; """
requeteSelect25_2MonetDB = """SELECT * FROM flights WHERE "DAY" > 25 AND "DAY_OF_WEEK"<=5 AND "SCHEDULED_DEPARTURE">1200; """
requeteSelect30_2MonetDB = """SELECT * FROM flights WHERE "DAY" > 30 AND "DAY_OF_WEEK"<=5 AND "SCHEDULED_DEPARTURE">1200; """
requeteSelect05_3MonetDB = """SELECT "FLIGHT_NUMBER" FROM flights WHERE "DAY" > 5;"""
requeteSelect10_3MonetDB = """SELECT "FLIGHT_NUMBER" FROM flights WHERE "DAY" > 10;"""
requeteSelect15_3MonetDB = """SELECT "FLIGHT_NUMBER" FROM flights WHERE "DAY" > 15;"""
requeteSelect20_3MonetDB = """SELECT "FLIGHT_NUMBER" FROM flights WHERE "DAY" > 20;"""
requeteSelect25_3MonetDB = """SELECT "FLIGHT_NUMBER" FROM flights WHERE "DAY" > 25;"""
requeteSelect30_3MonetDB = """SELECT "FLIGHT_NUMBER" FROM flights WHERE "DAY" > 30;"""
requeteCreateTable5MonetDB = """CREATE TABLE flights2 AS SELECT * FROM FLIGHTS WHERE "DAY" > 5;"""
requeteCreateTable10MonetDB = """CREATE TABLE flights2 AS SELECT * FROM FLIGHTS WHERE "DAY" > 10;"""
requeteCreateTable15MonetDB = """CREATE TABLE flights2 AS SELECT * FROM FLIGHTS WHERE "DAY" > 15;"""
requeteCreateTable20MonetDB = """CREATE TABLE flights2 AS SELECT * FROM FLIGHTS WHERE "DAY" > 20;"""
requeteCreateTable25MonetDB = """CREATE TABLE flights2 AS SELECT * FROM FLIGHTS WHERE "DAY" > 25;"""
requeteCreateTable30MonetDB = """CREATE TABLE flights2 AS SELECT * FROM FLIGHTS WHERE "DAY" > 30;"""
requeteDelete5MonetDB = """DELETE FROM flights WHERE "DAY" > 5;"""
requeteDelete10MonetDB = """DELETE FROM flights WHERE "DAY" > 10;"""
requeteDelete15MonetDB = """DELETE FROM flights WHERE "DAY" > 15;"""
requeteDelete20MonetDB = """DELETE FROM flights WHERE "DAY" > 20;"""
requeteDelete25MonetDB = """DELETE FROM flights WHERE "DAY" > 25;"""
requeteDelete30MonetDB = """DELETE FROM flights WHERE "DAY" > 30;"""
requeteInsertMonetDB = """INSERT INTO flights SELECT * FROM flights2;"""
requeteCopyTableMonetDB = """CREATE TABLE flights2 AS SELECT * FROM FLIGHTS;"""
requeteReplaceMonetDB = """UPDATE flights2 SET "WEATHER_DELAY" = -1 WHERE "WEATHER_DELAY" IS NULL;"""
requeteDropFlight2MonetDB = """DROP TABLE flights2;"""
#Postgre SQL
requeteSelect1PostgreSQL = """SELECT * FROM public.flights WHERE "AIRLINE" = 'AA'"""
requeteSelect2PostgreSQL = """SELECT * FROM public.flights"""
requeteSELECT3PostgreSQL = """SELECT * FROM flights WHERE "YEAR" = 2015 AND "MONTH" = 12 AND "DAY" = 32 AND "DAY_OF_WEEK" = 4 AND "AIRLINE" = 'B6' AND "FLIGHT_NUMBER" = 839"""
requeteSelect05PostgreSQL = """SELECT * FROM public.flights WHERE "DAY" > 5"""
requeteSelect10PostgreSQL = """SELECT * FROM public.flights WHERE "DAY" > 10"""
requeteSelect15PostgreSQL = """SELECT * FROM public.flights WHERE "DAY" > 15"""
requeteSelect20PostgreSQL = """SELECT * FROM public.flights WHERE "DAY" > 20"""
requeteSelect25PostgreSQL = """SELECT * FROM public.flights WHERE "DAY" > 25"""
requeteSelect30PostgreSQL = """SELECT * FROM public.flights WHERE "DAY" > 30"""
requeteSelect05_2PostgreSQL = """SELECT * FROM public.flights WHERE "DAY" > 5 AND "DAY_OF_WEEK"<=5 AND "SCHEDULED_DEPARTURE">1200 """
requeteSelect10_2PostgreSQL = """SELECT * FROM public.flights WHERE "DAY" > 10 AND "DAY_OF_WEEK"<=5 AND "SCHEDULED_DEPARTURE">1200 """
requeteSelect15_2PostgreSQL = """SELECT * FROM public.flights WHERE "DAY" > 15 AND "DAY_OF_WEEK"<=5 AND "SCHEDULED_DEPARTURE">1200 """
requeteSelect20_2PostgreSQL = """SELECT * FROM public.flights WHERE "DAY" > 20 AND "DAY_OF_WEEK"<=5 AND "SCHEDULED_DEPARTURE">1200 """
requeteSelect25_2PostgreSQL = """SELECT * FROM public.flights WHERE "DAY" > 25 AND "DAY_OF_WEEK"<=5 AND "SCHEDULED_DEPARTURE">1200 """
requeteSelect30_2PostgreSQL = """SELECT * FROM public.flights WHERE "DAY" > 30 AND "DAY_OF_WEEK"<=5 AND "SCHEDULED_DEPARTURE">1200 """
requeteSelect05_3PostgreSQL = """SELECT "FLIGHT_NUMBER" FROM public.flights WHERE "DAY" > 5"""
requeteSelect10_3PostgreSQL = """SELECT "FLIGHT_NUMBER" FROM public.flights WHERE "DAY" > 10"""
requeteSelect15_3PostgreSQL = """SELECT "FLIGHT_NUMBER" FROM public.flights WHERE "DAY" > 15"""
requeteSelect20_3PostgreSQL = """SELECT "FLIGHT_NUMBER" FROM public.flights WHERE "DAY" > 20"""
requeteSelect25_3PostgreSQL = """SELECT "FLIGHT_NUMBER" FROM public.flights WHERE "DAY" > 25"""
requeteSelect30_3PostgreSQL = """SELECT "FLIGHT_NUMBER" FROM public.flights WHERE "DAY" > 30"""
requeteCreateTable5PostgreSQL = """CREATE TABLE flights2 AS SELECT * FROM FLIGHTS WHERE "DAY" > 5"""
requeteCreateTable10PostgreSQL = """CREATE TABLE flights2 AS SELECT * FROM FLIGHTS WHERE "DAY" > 10"""
requeteCreateTable15PostgreSQL = """CREATE TABLE flights2 AS SELECT * FROM FLIGHTS WHERE "DAY" > 15"""
requeteCreateTable20PostgreSQL = """CREATE TABLE flights2 AS SELECT * FROM FLIGHTS WHERE "DAY" > 20"""
requeteCreateTable25PostgreSQL = """CREATE TABLE flights2 AS SELECT * FROM FLIGHTS WHERE "DAY" > 25"""
requeteCreateTable30PostgreSQL = """CREATE TABLE flights2 AS SELECT * FROM FLIGHTS WHERE "DAY" > 30"""
requeteDelete5PostgreSQL = """DELETE FROM flights WHERE "DAY" > 5"""
requeteDelete10PostgreSQL = """DELETE FROM flights WHERE "DAY" > 10"""
requeteDelete15PostgreSQL = """DELETE FROM flights WHERE "DAY" > 15"""
requeteDelete20PostgreSQL = """DELETE FROM flights WHERE "DAY" > 20"""
requeteDelete25PostgreSQL = """DELETE FROM flights WHERE "DAY" > 25"""
requeteDelete30PostgreSQL = """DELETE FROM flights WHERE "DAY" > 30"""
requeteInsertPostgreSQL = """INSERT INTO flights SELECT * FROM flights2"""
requeteCopyTablePostgreSQL = """CREATE TABLE flights2 AS SELECT * FROM FLIGHTS"""
requeteReplacePostgreSQL = """UPDATE flights SET "WEATHER_DELAY" = -1 WHERE "WEATHER_DELAY" IS NULL"""
requeteDropFlight2PostgreSQL = """DROP TABLE flights2"""
rapport : str = ""
def initMonetDB():
global rapport
reset_data_monetdb()
time = create_table_monetdb()
result = "[MonetDB] create table: time = " + str(time) + " s \n"
rapport = rapport + "\n" + result
print(result)
time = load_data_monetdb()
result = "[MonetDB] load data : time = " + str(time) + " s \n"
rapport = rapport + "\n" + result
print(result)
def execMonetDB(str):
return requete_monetdb(str)
def readMonetDB():
#execForMonetDB(5,[requeteSelect1MonetDB])
#execForMonetDB(5,[requeteSelect2MonetDB])
execForMonetDB(5,[requeteSELECT3MonetDB]) #ONE RESULT
#execForMonetDB(5,[requeteSelect05MonetDB])
#execForMonetDB(5,[requeteSelect10MonetDB])
#execForMonetDB(5,[requeteSelect15MonetDB])
#execForMonetDB(5,[requeteSelect20MonetDB])
#execForMonetDB(5,[requeteSelect25MonetDB])
#execForMonetDB(5,[requeteSelect30MonetDB])
#
#execForMonetDB(5,[requeteSelect05_2MonetDB])
#execForMonetDB(5,[requeteSelect10_2MonetDB])
#execForMonetDB(5,[requeteSelect15_2MonetDB])
#execForMonetDB(5,[requeteSelect20_2MonetDB])
#execForMonetDB(5,[requeteSelect25_2MonetDB])
#execForMonetDB(5,[requeteSelect30_2MonetDB])
#
#execForMonetDB(10,[requeteSelect05_3MonetDB])
#execForMonetDB(10,[requeteSelect10_3MonetDB])
#execForMonetDB(10,[requeteSelect15_3MonetDB])
#execForMonetDB(10,[requeteSelect20_3MonetDB])
#execForMonetDB(10,[requeteSelect25_3MonetDB])
#execForMonetDB(10,[requeteSelect30_3MonetDB])
def readAndWriteMonetDB():
execForMonetDB(4,[requeteCreateTable5MonetDB,requeteDelete5MonetDB,requeteInsertMonetDB,requeteDropFlight2MonetDB])
execForMonetDB(4,[requeteCreateTable10MonetDB,requeteDelete10MonetDB,requeteInsertMonetDB,requeteDropFlight2MonetDB])
execForMonetDB(4,[requeteCreateTable15MonetDB,requeteDelete15MonetDB,requeteInsertMonetDB,requeteDropFlight2MonetDB])
execForMonetDB(4,[requeteCreateTable20MonetDB,requeteDelete20MonetDB,requeteInsertMonetDB,requeteDropFlight2MonetDB])
execForMonetDB(4,[requeteCreateTable25MonetDB,requeteDelete25MonetDB,requeteInsertMonetDB,requeteDropFlight2MonetDB])
execForMonetDB(4,[requeteCreateTable30MonetDB,requeteDelete30MonetDB,requeteInsertMonetDB,requeteDropFlight2MonetDB])
#execForMonetDB(5,[requeteCopyTableMonetDB,requeteReplaceMonetDB,requeteDropFlight2MonetDB]) # 1.34 s
def randomWriteMonetDB(n):
global rapport
time = 0
min = 0
max = 0
for i in range(n):
requete = """INSERT INTO flights ("YEAR","MONTH","DAY","DAY_OF_WEEK","AIRLINE","FLIGHT_NUMBER","TAIL_NUMBER","ORIGIN_AIRPORT","DESTINATION_AIRPORT","SCHEDULED_DEPARTURE","DEPARTURE_TIME") VALUES ("""
requete += str(random.randint(2015,2016)) + "," + str(random.randint(1,12)) + "," + str(random.randint(1,31)) + "," + str(random.randint(1,7)) + ",'AA'," + str(random.randint(1,1000)) + ",'N1','JFK','LAX'," + str(random.randint(0,2400)) + "," + str(random.randint(0,2400)) + ");"
time += execMonetDB(requete)[0]
if(time<min or min == 0) :
min = time
if(time>max) :
max = time
time = time/n
result = "[MonetDB] requete (moyenne de " + str(n) +") : INSERT INTO flights random value \n\ttime = " + str(time) + " s" + "\n\tmin time = " + str(min) + "\n\tmax time = " + str(max)
rapport = rapport + "\n" + result
print(result)
def writeMonetDB():
randomWriteMonetDB(1000)
def runMonetDB():
readMonetDB()
#readAndWriteMonetDB()
#writeMonetDB()
#execute n fois les n requêtes
def execForMonetDB(n, tabReq):
global rapport
t = [[0] * len(tabReq) for _ in range(4)]
for i in range(n) :
print(i+1,"/",n)
for j in range(len(tabReq)) :
v = execMonetDB(tabReq[j])
t[0][j] += v[0]
t[1][j] = v[1]
if(v[0]<t[2][j] or t[2][j] == 0) :
t[2][j] = v[0]
if(v[0]>t[3][j]) :
t[3][j] = v[0]
print("calcul en cours...")
for j in range(len(tabReq)):
t[0][j] = t[0][j]/n
result = "[MonetDB] requete (moyenne de " + str(n) +") : " + str(tabReq[j]) + "\n\ttime = " + str(t[0][j]) + " s\n\tsize=" + str(t[1][j]) + "\n\tmin time = " + str(t[2][j]) + "\n\tmax time = " + str(t[3][j])
rapport = rapport + "\n" + result
print(result)
def initPostgreSQL():
global rapport
reset_database_postgres()
time = create_tables_postgres()
result = "[PostgreSQL] create table: time = " + str(time) + " s\n"
rapport = rapport + "\n" + result
print(result)
time = create_data_postgres()
result = "[PostgreSQL] load data : time = " + str(time) + " s\n"
rapport = rapport + "\n" + result
print(result)
def initIndex():
global rapport
time = execPostgreSQL("""CREATE INDEX idx_day ON flights ("DAY");""")
result = "[PostgreSQL] create index : time = " + str(time) + " s\n"
rapport = rapport + "\n" + result
print(result)
def execPostgreSQL(str):
return requete_postgres(str)
def readPostgreSQL():
#execForPostgreSQL(2,[requeteSelect1PostgreSQL])
#execForPostgreSQL(2,[requeteSelect2PostgreSQL])
execForPostgreSQL(5,[requeteSELECT3PostgreSQL]) #ONE RESULT
#execForPostgreSQL(2,[requeteSelect05PostgreSQL])
#execForPostgreSQL(2,[requeteSelect10PostgreSQL])
#execForPostgreSQL(2,[requeteSelect15PostgreSQL])
#execForPostgreSQL(2,[requeteSelect20PostgreSQL])
#execForPostgreSQL(2,[requeteSelect25PostgreSQL])
#execForPostgreSQL(3,[requeteSelect30PostgreSQL])
#
#execForPostgreSQL(2,[requeteSelect05_2PostgreSQL])
#execForPostgreSQL(2,[requeteSelect10_2PostgreSQL])
#execForPostgreSQL(2,[requeteSelect15_2PostgreSQL])
#execForPostgreSQL(2,[requeteSelect20_2PostgreSQL])
#execForPostgreSQL(2,[requeteSelect25_2PostgreSQL])
#execForPostgreSQL(2,[requeteSelect30_2PostgreSQL])
#
#execForPostgreSQL(5,[requeteSelect05_3PostgreSQL])
#execForPostgreSQL(5,[requeteSelect10_3PostgreSQL])
#execForPostgreSQL(5,[requeteSelect15_3PostgreSQL])
#execForPostgreSQL(5,[requeteSelect20_3PostgreSQL])
#execForPostgreSQL(5,[requeteSelect25_3PostgreSQL])
#execForPostgreSQL(5,[requeteSelect30_3PostgreSQL])
def readAndWritePostgreSQL():
execForPostgreSQL(4,[requeteCreateTable5PostgreSQL,requeteDelete5PostgreSQL,requeteInsertPostgreSQL,requeteDropFlight2PostgreSQL])
execForPostgreSQL(4,[requeteCreateTable10PostgreSQL,requeteDelete10PostgreSQL,requeteInsertPostgreSQL,requeteDropFlight2PostgreSQL])
execForPostgreSQL(4,[requeteCreateTable15PostgreSQL,requeteDelete15PostgreSQL,requeteInsertPostgreSQL,requeteDropFlight2PostgreSQL])
execForPostgreSQL(4,[requeteCreateTable20PostgreSQL,requeteDelete20PostgreSQL,requeteInsertPostgreSQL,requeteDropFlight2PostgreSQL])
execForPostgreSQL(4,[requeteCreateTable25PostgreSQL,requeteDelete25PostgreSQL,requeteInsertPostgreSQL,requeteDropFlight2PostgreSQL])
execForPostgreSQL(4,[requeteCreateTable30PostgreSQL,requeteDelete30PostgreSQL,requeteInsertPostgreSQL,requeteDropFlight2PostgreSQL])
#execForPostgreSQL(2,[requeteCopyTablePostgreSQL,requeteReplacePostgreSQL,requeteDropFlight2PostgreSQL]) # 11.97 s
def randomWritePostgreSQL(n):
global rapport
time = 0
min = 0
max = 0
for i in range(n):
requete = """INSERT INTO flights ("YEAR","MONTH","DAY","DAY_OF_WEEK","AIRLINE","FLIGHT_NUMBER","TAIL_NUMBER","ORIGIN_AIRPORT","DESTINATION_AIRPORT","SCHEDULED_DEPARTURE","DEPARTURE_TIME") VALUES ("""
requete += str(random.randint(2015,2016)) + "," + str(random.randint(1,12)) + "," + str(random.randint(1,31)) + "," + str(random.randint(1,7)) + ",'AA'," + str(random.randint(1,1000)) + ",'N1','JFK','LAX'," + str(random.randint(0,2400)) + "," + str(random.randint(0,2400)) + ");"
time += execPostgreSQL(requete)[0]
if(time<min or min == 0) :
min = time
if(time>max) :
max = time
time = time/n
result = "[PostgreSQL] requete (moyenne de " + str(n) +") : INSERT INTO flights random value \n\ttime = " + str(time) + " s" + "\n\tmin time = " + str(min) + "\n\tmax time = " + str(max)
rapport = rapport + "\n" + result
print(result)
def writePostgreSQL():
randomWritePostgreSQL(1000)
def runPostgreSQL():
readPostgreSQL()
#readAndWritePostgreSQL()
#writePostgreSQL()
# execute n fois les n requêtes
def execForPostgreSQL(n, tabReq):
global rapport
t = [[0] * len(tabReq) for _ in range(4)]
# avg nbResult min max
for i in range(n):
print(i+1,"/",n)
for j in range(len(tabReq)):
v = execPostgreSQL(tabReq[j])
t[0][j] += v[0]
t[1][j] = v[1]
if(v[0]<t[2][j] or t[2][j] == 0) :
t[2][j] = v[0]
if(v[0]>t[3][j]) :
t[3][j] = v[0]
print("calcul en cours...")
for j in range(len(tabReq)):
t[0][j] = t[0][j] / n
result = "[PostgreSQL] requete (moyenne de " + str(n) +") : " + str(tabReq[j]) + "\n\ttime = " + str(t[0][j]) + " s\n\tsize=" + str(t[1][j]) + "\n\tmin time = " + str(t[2][j]) + "\n\tmax time = " + str(t[3][j])
rapport = rapport + "\n" + result
print(result)
if __name__ == '__main__':
initMonetDB()
runMonetDB()
rapport += "\n"
initPostgreSQL()
#initIndex()
runPostgreSQL()
print("/////////////////////////////////////////////////////////////")
print(rapport)
print("/////////////////////////////////////////////////////////////")
date = datetime.now()
nom_fichier = "result" + date.strftime("_%d_%m_%y_%H_%M_%S") + ".txt" #un nom de fichier unique pour ne pas écraser les anciens résultats
with open(nom_fichier, 'w') as fichier:
fichier.write(rapport)
#