Permalink
Cannot retrieve contributors at this time
chatbot-jeff/DataAPI.py
Go to file| # Gets JSON file data from a url | |
| def readJSON(url): # Andre | |
| ''' Gets JSON file data from a url ''' | |
| import urllib.request | |
| import json | |
| try: | |
| page = urllib.request.urlopen(url) | |
| except urllib.error.URLError as e: | |
| return("Error", 0) | |
| data = page.read() | |
| data.decode('utf-8') | |
| page.close() | |
| try: | |
| jsonData = json.loads(data) | |
| except: | |
| return("Error", 1) | |
| return(jsonData) | |
| # If desn't exist, ask for a new token and save it on the file, if it does, then | |
| # read it from the file | |
| def tokenGetSet(): # Andre | |
| ''' Gets the token from the file, or requests a new one, outputs the token ''' | |
| from pathlib import Path | |
| if (Path("./data/token.jeff").exists()): | |
| with open('./data/token.jeff', "r") as tkFile: | |
| tkReq = tkFile.read() | |
| if (tkReq == "" or tkReq is None): | |
| return("Error", 7) | |
| else: | |
| tkJSON = readJSON("https://opentdb.com/api_token.php?command=request") | |
| if ("Error" in tkJSON): | |
| return(tkJSON) | |
| tkReq = tkJSON["token"] | |
| Path('./data').mkdir(parents=True, exist_ok=True) | |
| with open('./data/token.jeff', "w") as tkFile: | |
| tkFile.write(tkReq) | |
| return(tkReq) | |
| # Checks OpentDB categories and output them and the respective ID, if input true | |
| # only output the categories names | |
| def getCategories(onlyCategories = False, nrCat = 0): # Andre | |
| ''' Gets opentDB possibible categories and ID, if true just show the categories ''' | |
| import html | |
| import random | |
| if (onlyCategories != False and onlyCategories != True): | |
| return("Error", 6) | |
| if (nrCat < 0): | |
| return("Error", 6) | |
| dbcat = readJSON('https://opentdb.com/api_category.php') | |
| if ("Error" in dbcat): | |
| return(dbcat) | |
| categories = [] | |
| if (onlyCategories): | |
| for i in dbcat["trivia_categories"]: | |
| categories.append(html.unescape(i["name"].replace("Entertainment: ","") | |
| .replace("Science: ", ""))) | |
| else: | |
| for i in dbcat["trivia_categories"]: | |
| categories.append(html.unescape(i)) | |
| if (nrCat > 0 and nrCat < len(categories)): | |
| tempCat = categories | |
| categories = random.sample(tempCat, nrCat) | |
| return(categories) | |
| # Gets questions from OpentDB, with defined criteria | |
| def getOpentDB(category, difficulty, nrQuestions = 1): # Andre | |
| ''' With a set number of questions, category and difficulty level | |
| outputs the questions with that criteria ''' | |
| import html | |
| import json | |
| import random | |
| ### Check if the number of questions is valid, more than 0 and less than 50 | |
| if (nrQuestions is None or nrQuestions == "" | |
| or nrQuestions < 1 or nrQuestions > 50): | |
| return("Error" , 2) | |
| ### Get possible categories, check for validity and respective ID | |
| if (category is None): | |
| return("Error" , 3) | |
| if (category.casefold() not in ["", "Any", "Random"]): | |
| cat = getCategories() | |
| if ("Error" in cat): | |
| return(cat) | |
| catID = None | |
| for i in cat: | |
| if (category.casefold() in i["name"].casefold()): | |
| catID = "&category={}".format(i["id"]) | |
| break | |
| if (catID == None): | |
| return("Error" , 3) | |
| else: | |
| catID = "" | |
| ### Check difficulty | |
| if (difficulty is None): | |
| return("Error" , 4) | |
| if (difficulty.casefold() not in ["", "Any", "Random"]): | |
| if (difficulty.casefold() in ["easy","medium","hard"]): | |
| diffID = "&difficulty={}".format(difficulty) | |
| else: | |
| return("Error" , 4) | |
| else: | |
| diffID = "" | |
| # Get token | |
| token = tokenGetSet() | |
| if ("Error" in token): | |
| return(token) | |
| else: | |
| tokenID = "&token={}".format(token) | |
| # Get JSON data from url with defined criteria | |
| opentDBJson = readJSON( | |
| "https://opentdb.com/api.php?amount={}{}{}{}" | |
| .format(nrQuestions, catID, diffID, tokenID)) | |
| if ("Error" in opentDBJson): | |
| return(opentDBJson) | |
| # Check website API response message | |
| if (opentDBJson["response_code"] == 0): # Success - Read the data into variables | |
| data = opentDBJson["results"][0] | |
| questionSet = [] | |
| for s in opentDBJson["results"]: | |
| setTemp = {} | |
| setTemp["Type"] = html.unescape(s["type"]).title() | |
| setTemp["Question"] = html.unescape(s["question"]) | |
| setTemp["corrAnswer"] = html.unescape(s["correct_answer"]) | |
| setTemp["corrAnswer"] = html.unescape(s["correct_answer"]) | |
| if (setTemp["Type"] == "Multiple"): | |
| choices = ["A", "B", "C", "D"] | |
| ansTemp = [html.unescape(s["correct_answer"]), | |
| html.unescape(s["incorrect_answers"][0]), | |
| html.unescape(s["incorrect_answers"][1]), | |
| html.unescape(s["incorrect_answers"][2])] | |
| else: | |
| choices = ["A", "B"] | |
| ansTemp = [html.unescape(s["correct_answer"]), | |
| html.unescape(s["incorrect_answers"][0])] | |
| random.shuffle(ansTemp) | |
| for i, item in enumerate(ansTemp): | |
| setTemp[choices[i]] = item | |
| questionSet.append(setTemp) | |
| elif (opentDBJson["response_code"] == 1): #No Results | |
| return("Error", 5) | |
| elif (opentDBJson["response_code"] == 2): #Invalid Parameter | |
| return("Error", 6) | |
| elif (opentDBJson["response_code"] >= 3): #Token Not Found (3) and Token Empty (4) | |
| return("Error", 7) | |
| return(questionSet) | |
| def getBirthday(): # Andre | |
| ''' Get a random birthday and output a dictionairy with name as string, | |
| the date as string and a questions and righ answer as strings ''' | |
| import json | |
| import random | |
| # Get JSON data from portfolio website file | |
| birthdaysJson = readJSON("http://andrefmsilva.coventry.domains/ALL%20Project%20One%20-%20Chatbot%20Data/Birthdays.json") | |
| if ("Error" in birthdaysJson): | |
| return(birthdaysJson) | |
| chooseBirthday = random.choice(birthdaysJson) | |
| birthdaySet = [{"Type": "Birthday", "Name": chooseBirthday["Name"], "Date": chooseBirthday["Date"], | |
| "Question": "In what year was " + chooseBirthday["Name"] + " born?", "corrAnswer": chooseBirthday["Date"].split("-")[0]}] | |
| return(birthdaySet) | |
| def getHistory(): # Andre | |
| ''' Get a random history event and output a dictionairy with the event as string, | |
| the date as string and a questions and righ answer as strings ''' | |
| import json | |
| import random | |
| # Get JSON data from portfolio website file | |
| historyJson = readJSON("http://andrefmsilva.coventry.domains/ALL%20Project%20One%20-%20Chatbot%20Data/History.json") | |
| if ("Error" in historyJson): | |
| return(historyJson) | |
| chooseHistory = random.choice(historyJson) | |
| historySet = [{"Type": "History", "Event": chooseHistory["Event"], "Date": chooseHistory["Date"], | |
| "Question": chooseHistory["Event"] + " in what year?", "corrAnswer": chooseHistory["Date"].split("-")[0]}] | |
| return(historySet) | |
| def getQuote(): # Andre | |
| ''' Get a random quote and output a dictionairy with the quotetype as string, | |
| the name and quote as string and a questions and righ answer as strings ''' | |
| import json | |
| import random | |
| # Get JSON data from portfolio website file | |
| quoteJson = readJSON("http://andrefmsilva.coventry.domains/ALL%20Project%20One%20-%20Chatbot%20Data/Quotes.json") | |
| if ("Error" in quoteJson): | |
| return(quoteJson) | |
| chooseQuote = random.choice(quoteJson) | |
| quoteSet = [{"Type": "Quote", "QuoteType": chooseQuote["Type"], "Name": chooseQuote["Name"], "Quote": chooseQuote["Quote"], | |
| "Question": "Who said '" + chooseQuote["Quote"] + "' ?", "corrAnswer": chooseQuote["Name"]}] | |
| return(quoteSet) | |
| # Main function to get questions, choose database and check for the error messages | |
| def getQuestion(category, difficulty, nrQuestions = 1, qSource = "OpentDB"): # Andre | |
| ''' Given a category, diffculty as strings and number | |
| of questions as integer return a dictionary with | |
| type, question, possible answers and correct answer ''' | |
| if (qSource == "OpentDB"): | |
| questionSet = getOpentDB(category, difficulty, nrQuestions) | |
| elif (qSource == "Birthday"): | |
| questionSet = getBirthday() | |
| elif (qSource == "History"): | |
| questionSet = getHistory() | |
| elif (qSource == "Quote"): | |
| questionSet = getQuote() | |
| if ("Error" in questionSet): # Return every thing as a list to keep the same structure as getting multiple questions | |
| if (questionSet[1] == 0): | |
| return([("Error", "There was an error opening the URL")]) | |
| elif (questionSet[1] == 1): | |
| return([("Error", "Error reading JSON data")]) | |
| elif (questionSet[1] == 2): | |
| return([("Error", "The number of questions requested not valid")]) | |
| elif (questionSet[1] == 3): | |
| return([("Error", "Category not valid")]) | |
| elif (questionSet[1] == 4): | |
| return([("Error", "Difficulty not valid")]) | |
| elif (questionSet[1] == 5): | |
| return([("Error", "No results were found")]) | |
| elif (questionSet[1] == 6): | |
| return([("Error", "Invalid parameter")]) | |
| elif (questionSet[1] == 7): | |
| import os | |
| print("Token not found or all available question were used") | |
| print("Reseting token...") | |
| os.remove("./data/token.jeff") | |
| questionSet = getOpentDB(category, difficulty, nrQuestions) | |
| if ("Error" in questionSet): | |
| return([("Error", "Token not found or all available question were used")]) | |
| return(questionSet) | |
| def getGoogleSearch(toSearch): # Andre | |
| ''' With a input search as string, return the first google search as a string''' | |
| search = readJSON("https://www.googleapis.com/customsearch/v1?q={}&cx=001954664739637419008%3Aprvczty2gta&num=1&safe=medium&key=AIzaSyArPcOPqon_MSxFKkB41qCexrCVZ5AHfoU" | |
| .format(toSearch.replace(" ", "+")))["items"][0] | |
| snippet = search["snippet"].replace("\xa0...",".").replace("\n","") | |
| url = search["link"] | |
| return(snippet, url) | |
| ######################################################## | |
| # # | |
| # Testing area # | |
| # # | |
| ######################################################## | |
| if (__name__ == "__main__"): | |
| import json | |
| import os | |
| import time | |
| import html | |
| from pathlib import Path | |
| ########## Test readJSON function ########## | |
| print("#" * 50) | |
| print("# Checking readJSON function, read JSON from url") | |
| time_Start = time.perf_counter() | |
| # Test 1 - If JSON had the required fields | |
| test_Json = readJSON("https://opentdb.com/api.php?amount=1") | |
| print(" - Read JSON: ", end="") | |
| test_Error = (False if ("response_code" in test_Json and | |
| "results" in test_Json and | |
| "category" in test_Json["results"][0] and | |
| "correct_answer" in test_Json["results"][0] and | |
| "difficulty" in test_Json["results"][0] and | |
| "incorrect_answers" in test_Json["results"][0] and | |
| "question" in test_Json["results"][0] and | |
| "type" in test_Json["results"][0]) else True) | |
| print("OK" if not test_Error else "NOT OK\n{}".format(test_Json)) | |
| print("# Ran 1 test in {}s".format(round(time.perf_counter() - time_Start, 3))) | |
| ########## Test tokenGetSet function ########## | |
| print() | |
| print("#" * 50) | |
| print("# Checking tokenGetSet function, get a token, store an read it") | |
| if (Path("./data/token.jeff").exists()): | |
| os.rename("./data/token.jeff", "./data/temp_tk.jeff") | |
| time_Start = time.perf_counter() | |
| # Test 1 - If token works on the website API | |
| print(" - Get token: ", end="") | |
| test_Token = tokenGetSet() | |
| test_UseToken = readJSON("https://opentdb.com/api.php?amount=1&token={}" | |
| .format(test_Token)) | |
| test_Error = (False if ("Error" not in test_UseToken and | |
| test_UseToken["response_code"] != 3) else True) | |
| print("OK" if not test_Error else "NOT OK\n{}".format(test_Token)) | |
| # Test 2 - If token was kept in file, and if function reads it | |
| print(" - Read token: ", end="") | |
| test_tokenFile = "Error" | |
| if (Path("./data/token.jeff").exists()): | |
| with open('./data/token.jeff', "r") as test_TkFile: | |
| test_tokenFile = test_TkFile.read() | |
| print("OK" if test_Token == test_tokenFile else "NOT OK\n{}".format(test_Token)) | |
| print("# Ran 2 tests in {}s".format(round(time.perf_counter() - time_Start, 3))) | |
| if (Path("./data/token.jeff").exists()): | |
| os.remove("./data/token.jeff") | |
| if (Path("./data/temp_tk.jeff").exists()): | |
| os.rename("./data/temp_tk.jeff", "./data/token.jeff") | |
| ########## Test getCategories function ########## | |
| print() | |
| print("#" * 50) | |
| print("# Checking getCategories function, get possible categories") | |
| time_Start = time.perf_counter() | |
| # Test 1 - If categories has the required fields | |
| test_categories = getCategories(); | |
| print(" - Get Categories: ", end="") | |
| test_Error = (False if ("id" in test_categories[0] and | |
| "name" in test_categories[0]) else True) | |
| print("OK" if not test_Error else "NOT OK\n{}".format(test_categories)) | |
| # Test 2 - If function sorts only the categories | |
| print(" - Only Categories: ", end="") | |
| test_temp_cat = [] | |
| if ("Error" not in test_categories): | |
| for i in test_categories: | |
| test_temp_cat.append(html.unescape(i["name"] | |
| .replace("Entertainment: ","") | |
| .replace("Science: ", ""))) | |
| test_onlyCategories = getCategories(True); | |
| print("OK" if (test_onlyCategories == test_temp_cat) | |
| else "NOT OK\n{}".format(test_onlyCategories)) | |
| # Test 3 - If returns a set number of random categories | |
| test_nrCat = [-30, -24, -10, -1, 0, 1, 10, 24, 30] | |
| print(" - Random Categories: ", end="") | |
| for i in test_nrCat: | |
| test_randomCat = getCategories(True, i) | |
| test_Error = (False if (test_randomCat in test_onlyCategories or | |
| test_randomCat == test_onlyCategories) else True) | |
| print("OK" if not test_Error else "NOT OK\n{}".format(test_randomCat)) | |
| print("# Ran 2 tests in {}s".format(round(time.perf_counter() - time_Start, 3))) | |
| ########## Test getOpentDB function ########## | |
| print() | |
| print("#" * 50) | |
| print("# Checking getOpentDB function, get data from OpentDB") | |
| time_Start = time.perf_counter() | |
| # Test 1 - If function gets random data | |
| print(" - Random data: ", end="") | |
| test_Data = getQuestion("", "")[0] | |
| test_Error = (False if ("Type" in test_Data and "Question" in test_Data and | |
| "A" in test_Data and "B" in test_Data and | |
| "corrAnswer" in test_Data) else True) | |
| print("{}".format("OK" if not test_Error else "NOT OK\n{}".format(test_Data))) | |
| # Test 2 - If function gets all possible categories | |
| print(" - All Categories: ", end="") | |
| test_Error = False | |
| test_catList = getCategories(True) | |
| if ("Error" in test_catList): | |
| test_Error = True | |
| test_Cat = test_catList | |
| else: | |
| for i in test_catList: | |
| test_Data = getQuestion(i, "")[0] | |
| if ("Type" not in test_Data or "Question" not in test_Data or | |
| "A" not in test_Data or "B" not in test_Data or | |
| "corrAnswer" not in test_Data): | |
| test_Error = True | |
| break | |
| print("OK" if not test_Error else "NOT OK\n{}".format(test_Data)) | |
| # Test 3 - If function gets all difficulties | |
| print(" - All Difficulties: ", end="") | |
| test_Error = False | |
| for i in ["easy", "medium", "hard"]: | |
| test_Data = getQuestion("", i)[0] | |
| if ("Type" not in test_Data or "Question" not in test_Data or | |
| "A" not in test_Data or "B" not in test_Data or | |
| "corrAnswer" not in test_Data): | |
| test_Error = True | |
| break | |
| print("OK" if not test_Error else "NOT OK\n{}".format(test_Data)) | |
| # Test 4 - If function possible number of questions Input | |
| print(" - Number Questions: ", end="") | |
| test_Error = False | |
| test_nrQuest = [-100, -50, -1, 0, 1, 50, 100] | |
| for i in test_nrQuest: | |
| test_Data = getQuestion("", "", i) | |
| if (i == 1): | |
| test_Error = (False if ("Type" in test_Data[0] | |
| and "Question" in test_Data[0] and | |
| "A" in test_Data and "B" in test_Data[0] and | |
| "corrAnswer" in test_Data[0]) else True) | |
| else: | |
| if (len(test_Data) != i and test_Data[0] != "Error"): | |
| test_Error = True | |
| break | |
| elif (test_Data[0] != "Error"): | |
| for i in test_Data: | |
| test_Error = (False if ("Type" in i and "Question" in i and | |
| "A" in i and "B" in i and | |
| "corrAnswer" in i) else True) | |
| print("OK" if not test_Error else "NOT OK\n{}".format(test_Data)) | |
| print("# Ran 4 test in {}s".format(round(time.perf_counter() - time_Start, 3))) | |
| ########## Test getBirthday function ########## | |
| print() | |
| print("#" * 50) | |
| print("# Checking getBirthday function, get data from a JSON file on portfolio website") | |
| time_Start = time.perf_counter() | |
| print(" - Get Birthday: ", end="") | |
| test_Error = False | |
| test_Data = getBirthday()[0] | |
| test_Error = (False if ("Type" in test_Data and | |
| "Name" in test_Data and | |
| "Date" in test_Data and | |
| "Question" in test_Data and | |
| "corrAnswer" in test_Data) else True) | |
| print("OK" if not test_Error else "NOT OK\n{}".format(test_Data)) | |
| print("# Ran 4 test in {}s".format(round(time.perf_counter() - time_Start, 3))) | |
| ########## Test getHistory function ########## | |
| print() | |
| print("#" * 50) | |
| print("# Checking getHistory function, get data from a JSON file on portfolio website") | |
| time_Start = time.perf_counter() | |
| print(" - Get History: ", end="") | |
| test_Error = False | |
| test_Data = getHistory()[0] | |
| test_Error = (False if ("Type" in test_Data and | |
| "Event" in test_Data and | |
| "Date" in test_Data and | |
| "Question" in test_Data and | |
| "corrAnswer" in test_Data) else True) | |
| print("OK" if not test_Error else "NOT OK\n{}".format(test_Data)) | |
| print("# Ran 4 test in {}s".format(round(time.perf_counter() - time_Start, 3))) | |
| ########## Test getQuote function ########## | |
| print() | |
| print("#" * 50) | |
| print("# Checking getQuote function, get data from a JSON file on portfolio website") | |
| time_Start = time.perf_counter() | |
| print(" - Get Quote: ", end="") | |
| test_Error = False | |
| test_Data = getQuote()[0] | |
| test_Error = (False if ("Type" in test_Data and | |
| "Name" in test_Data and | |
| "Quote" in test_Data and | |
| "Question" in test_Data and | |
| "corrAnswer" in test_Data) else True) | |
| print("OK" if not test_Error else "NOT OK\n{}".format(test_Data)) | |
| print("# Ran 4 test in {}s".format(round(time.perf_counter() - time_Start, 3))) | |
| ########## Test getQuestion function ########## | |
| print() | |
| print("#" * 50) | |
| print("# Checking getQuestion function, get questions from the choosen type and check for errors") | |
| time_Start = time.perf_counter() | |
| # Test 1 - OpentDB with one question | |
| print(" - OpentDB 1 Question: ", end="") | |
| test_Error = False | |
| test_Data = getQuestion("", "", 1, "OpentDB")[0] | |
| test_Error = (False if ("Type" in test_Data and | |
| "Question" in test_Data and | |
| "A" in test_Data and | |
| "B" in test_Data and | |
| "corrAnswer" in test_Data) else True) | |
| print("OK" if not test_Error else "NOT OK\n{}".format(test_Data)) | |
| # Test 2 - OpentDB with ten question | |
| print(" - OpentDB 10 Question: ", end="") | |
| test_Error = False | |
| test_Data = getQuestion("", "", 10, "OpentDB") | |
| for q in test_Data: | |
| if ("Type" not in q or "Question" not in q or "A" not in q or "B" not in q or "corrAnswer" not in q): | |
| test_Error = True | |
| break | |
| print("OK" if not test_Error else "NOT OK\n{}".format(test_Data)) | |
| # Test 3, 4 and 5 - Birthday, History and Quote | |
| for source in ["Birthday", "History", "Quote"]: | |
| print(" - {} Question: ".format(source), end="") | |
| test_Error = False | |
| test_Data = getQuestion("", "", 1, source)[0] | |
| test_Error = (False if ("Type" in test_Data and | |
| "Question" in test_Data and | |
| "corrAnswer" in test_Data) else True) | |
| print("OK" if not test_Error else "NOT OK\n{}".format(test_Data)) | |
| print("# Ran 4 test in {}s".format(round(time.perf_counter() - time_Start, 3))) | |
| ########## Test getGoogleSearch function ########## | |
| print() | |
| print("#" * 50) | |
| print("# Checking getGoogleSearch function, gets the first search from google") | |
| time_Start = time.perf_counter() | |
| # Test 1 - Get a random search | |
| print(" - Get Search: ", end="") | |
| test_Error = False | |
| test_Data = getGoogleSearch("Hi") | |
| test_Error = (False if (type(test_Data) == tuple) else True) | |
| print("OK" if not test_Error else "NOT OK\n{}".format(test_Data)) | |
| # Test 2 - Get known search result 1 | |
| print(" - What is pizza: ", end="") | |
| test_Error = False | |
| test_Data = getGoogleSearch("What is pizza") | |
| test_Error = (False if (test_Data[0] == "Pizza is a yeasted flatbread typically topped with tomato sauce and cheese and baked in an oven. It is commonly topped with a selection of meats, vegetables." and | |
| test_Data[1] == "https://en.wikipedia.org/wiki/Pizza") else True) | |
| print("OK" if not test_Error else "NOT OK\n{}".format(test_Data)) | |
| # Test 3 - Get known search result 1 | |
| print(" - How train works: ", end="") | |
| test_Error = False | |
| test_Data = getGoogleSearch("How train works") | |
| test_Error = (False if (test_Data[0] == "Trains act as a major form of transportation worldwide. Learn how trains evolved from horse-drawn carts to the high-speed sleek railways of today." and | |
| test_Data[1] == "https://science.howstuffworks.com/transport/engines-equipment/train.htm") else True) | |
| print("OK" if not test_Error else "NOT OK\n{}".format(test_Data)) | |
| print("# Ran 4 test in {}s".format(round(time.perf_counter() - time_Start, 3))) |