try1.py

#Code created using this documentation: https://docs.python.org/3/library/tokenize.html
from tokenizer import tokenize, TOK

token = ["chicken","bacon"]         #Added example foods, to test if tokenizer works.
food = input("Enter ingredient ")

for token in tokenize(food):

    print(u"{0}: '{1}' {2}".format(
        TOK.descr[token.kind],
        token.txt or "-",
        token.val or ""))