Guía paso a paso para crear un chatbot personalizado utilizando spaCy (librería Python NLP)

Hola Comunidad,

En este artículo, os demostraré los siguientes pasos para crear vuestro propio chatbot utilizando spaCy (spaCy es una biblioteca de software de código abierto para el procesamiento avanzado del lenguaje natural, escrita en los lenguajes de programación Python y Cython):

• Paso 1: Instalar las librerías necesarias

• Paso2: Crear el archivo de patrones y respuestas

• Paso 3: Entrenar el modelo

• Paso 4: Crear una aplicación ChatBot basada en el modelo entrenado

Empecemos

Paso 1: Instalar las librerías necesarias

En primer lugar, necesitamos instalar las librerías python necesarias, lo que podemos conseguir ejecutando el siguiente comando:

pip3 install spacy nltk
pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu

Paso2: Crear el archivo de patrones y respuestas

Necesitamos crear un archivo intents.json que contenga patrones de preguntas y respuestas, a continuación se muestra un ejemplo de algunos de los patrones de preguntas y respuestas

{
  "intents": [
    {
      "tag": "greeting",
      "patterns": [
        "Hi",
        "Hey",
        "How are you",
        "Is anyone there?",
        "Hello",
        "Good day"
      ],
      "responses": [
        "Hey :-)",
        "Hello, thanks for visiting",
        "Hi there, what can I do for you?",
        "Hi there, how can I help?"
      ]
    },
    {
      "tag": "goodbye",
      "patterns": ["Bye", "See you later", "Goodbye"],
      "responses": [
        "See you later, thanks for visiting",
        "Have a nice day",
        "Bye! Come back again soon."
      ]
    },
    {
      "tag": "thanks",
      "patterns": ["Thanks", "Thank you", "That's helpful", "Thank's a lot!"],
      "responses": ["Happy to help!", "Any time!", "My pleasure"]
    },
    {
      "tag": "items",
      "patterns": [
        "tell me about this app",
        "What kinds of technology used?",
        "What do you have?"
      ],
      "responses": [
        "Write something about the app."
      ]
    },        
    {
      "tag": "funny",
      "patterns": [
        "Tell me a joke!",
        "Tell me something funny!",
        "Do you know a joke?"
      ],
      "responses": [
        "Why did the hipster burn his mouth? He drank the coffee before it was cool.",
        "What did the buffalo say when his son left for college? Bison."
      ]
    }
  ]
}

Paso 3: Entrenar el modelo

3.1- Crear un fichero NeuralNet model.py que se utilizará para entrenar el modelo

#model.py fileimport torch.nn as nn

classNeuralNet(nn.Module):def__init__(self, input_size, hidden_size, num_classes):
        super(NeuralNet, self).__init__()
        self.l1 = nn.Linear(input_size, hidden_size) 
        self.l2 = nn.Linear(hidden_size, hidden_size) 
        self.l3 = nn.Linear(hidden_size, num_classes)
        self.relu = nn.ReLU()
    
    defforward(self, x):
        out = self.l1(x)
        out = self.relu(out)
        out = self.l2(out)
        out = self.relu(out)
        out = self.l3(out)
        # no activation and no softmax at the endreturn out

3.2 - Necesitamos el archivo nltk_utils.py que utiliza la librería python «nltk» (una plataforma utilizada para construir programas Python que trabajan con datos del lenguaje humano para aplicarlos en el procesamiento estadístico del lenguaje natural (PLN)). 

#nltk_utils.py#nltk library used for building Python programs that work with human language data for applying in #statistical natural language processing (NLP)import numpy as np
import nltk
from nltk.stem.porter import PorterStemmer
stemmer = PorterStemmer()

deftokenize(sentence):"""
    split sentence into array of words/tokens
    a token can be a word or punctuation character, or number
    """return nltk.word_tokenize(sentence)


defstem(word):"""
    stemming = find the root form of the word
    examples:
    words = ["organize", "organizes", "organizing"]
    words = [stem(w) for w in words]
    -> ["organ", "organ", "organ"]
    """return stemmer.stem(word.lower())


defbag_of_words(tokenized_sentence, words):"""
    return bag of words array:
    1 for each known word that exists in the sentence, 0 otherwise
    example:
    sentence = ["hello", "how", "are", "you"]
    words = ["hi", "hello", "I", "you", "bye", "thank", "cool"]
    bog   = [  0 ,    1 ,    0 ,   1 ,    0 ,    0 ,      0]
    """# stem each word
    sentence_words = [stem(word) for word in tokenized_sentence]
    # initialize bag with 0 for each word
    bag = np.zeros(len(words), dtype=np.float32)
    for idx, w in enumerate(words):
        if w in sentence_words: 
            bag[idx] = 1return bag

3.3 - Tenemos que entrenar el modelo, vamos a utilizar el archivo train.py para crear nuestro modelo

#train.pyimport numpy as np
import json,torch
import torch.nn as nn
from torch.utils.data import Dataset, DataLoader
from nltk_utils import bag_of_words, tokenize, stem
from model import NeuralNet
 
with open('intents.json', 'r') as f:
    intents = json.load(f)

all_words = []
tags = []
xy = []
# loop through each sentence in our intents patternsfor intent in intents['intents']:
    tag = intent['tag']
    # add to tag list
    tags.append(tag)
    for pattern in intent['patterns']:
        # tokenize each word in the sentence
        w = tokenize(pattern)
        # add to our words list
        all_words.extend(w)
        # add to xy pair
        xy.append((w, tag))

# stem and lower each word
ignore_words = ['?', '.', '!']
all_words = [stem(w) for w in all_words if w notin ignore_words]
# remove duplicates and sort
all_words = sorted(set(all_words))
tags = sorted(set(tags))

print(len(xy), "patterns")
print(len(tags), "tags:", tags)
print(len(all_words), "unique stemmed words:", all_words)

# create training data
X_train = []
y_train = []
for (pattern_sentence, tag) in xy:
    # X: bag of words for each pattern_sentence
    bag = bag_of_words(pattern_sentence, all_words)
    X_train.append(bag)
    # y: PyTorch CrossEntropyLoss needs only class labels, not one-hot
    label = tags.index(tag)
    y_train.append(label)

X_train = np.array(X_train)
y_train = np.array(y_train)

# Hyper-parameters 
num_epochs = 1000
batch_size = 8
learning_rate = 0.001
input_size = len(X_train[0])
hidden_size = 8
output_size = len(tags)
print(input_size, output_size)

classChatDataset(Dataset):def__init__(self):
        self.n_samples = len(X_train)
        self.x_data = X_train
        self.y_data = y_train

    # support indexing such that dataset[i] can be used to get i-th sampledef__getitem__(self, index):return self.x_data[index], self.y_data[index]

    # we can call len(dataset) to return the sizedef__len__(self):return self.n_samples

dataset = ChatDataset()
train_loader = DataLoader(dataset=dataset,
                          batch_size=batch_size,
                          shuffle=True,
                          num_workers=0)

device = torch.device('cuda'if torch.cuda.is_available() else'cpu')
model = NeuralNet(input_size, hidden_size, output_size).to(device)
# Loss and optimizer
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
# Train the modelfor epoch in range(num_epochs):
    for (words, labels) in train_loader:
        words = words.to(device)
        labels = labels.to(dtype=torch.long).to(device)        
        # Forward pass
        outputs = model(words)
        # if y would be one-hot, we must apply# labels = torch.max(labels, 1)[1]
        loss = criterion(outputs, labels)        
        # Backward and optimize
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()        
    if (epoch+1) % 100 == 0:
        print (f'Epoch [{epoch+1}/{num_epochs}], Loss: {loss.item():.4f}')

print(f'final loss: {loss.item():.4f}')
data = {
"model_state": model.state_dict(),
"input_size": input_size,
"hidden_size": hidden_size,
"output_size": output_size,
"all_words": all_words,
"tags": tags
}
FILE = "chatdata.pth"
torch.save(data, FILE)
print(f'training complete. file saved to {FILE}')

3.4 - Ejecute el archivo train.py anterior para entrenar el modelo

El archivo train.py creará el archivo chatdata.pth que se utilizará en nuestro archivo chat.py

Paso 4: Crear una aplicación ChatBot basada en el modelo entrenado

Como Nuestro modelo es creado. En el paso final, vamos a crear un archivo chat.py que podemos utilizar en nuestro chatbot.

#chat.pyimport os,random,json,torch
from model import NeuralNet
from nltk_utils import bag_of_words, tokenize

device = torch.device('cuda'if torch.cuda.is_available() else'cpu')

with open("intents.json") as json_data:
    intents = json.load(json_data)

data_dir = os.path.join(os.path.dirname(__file__))
FILE = os.path.join(data_dir, 'chatdata.pth')
data = torch.load(FILE)

input_size = data["input_size"]
hidden_size = data["hidden_size"]
output_size = data["output_size"]
all_words = data['all_words']
tags = data['tags']
model_state = data["model_state"]

model = NeuralNet(input_size, hidden_size, output_size).to(device)
model.load_state_dict(model_state)
model.eval()

bot_name = "iris-NLP"defget_response(msg):
    sentence = tokenize(msg)
    X = bag_of_words(sentence, all_words)
    X = X.reshape(1, X.shape[0])
    X = torch.from_numpy(X).to(device)

    output = model(X)
    _, predicted = torch.max(output, dim=1)

    tag = tags[predicted.item()]

    probs = torch.softmax(output, dim=1)
    prob = probs[0][predicted.item()]
    if prob.item() > 0.75:
        for intent in intents['intents']:
            if tag == intent["tag"]:
                return random.choice(intent['responses'])
    return"I do not understand..."if __name__ == "__main__":
    print("Let's chat! (type 'quit' to exit)")
    whileTrue:
        sentence = input("You: ")
        if sentence == "quit":
            break
        resp = get_response(sentence)
        print(resp)

Ejecutad el archivo chat.py y realizad algunas preguntas que ya definimos anteriormente en nuestro archivo intents.json

 

Para más información, visitad la página de solicitud de intercambio abierto IRIS-GenLab
Gracias