Парсер и нормализованный текст

import requests
import csv
 
 
def get_posts(group_url):
    token = 'f379b0f2f379b0f2f379b0f288f06e696bff379f379b0f29574d1100fc7348fa33f1184'
    posts = []
    offset = 0
 
    while offset < 200:
        url = f'https://api.vk.com/method/wall.get?domain={group_url}&access_token={token}&v=5.92'
        response = requests.get(url, params={
            'count': 200,
            'offset': offset
        }
                                )
        data = response.json()
 
 
        for item in data['response']['items']:
            post_id = item['id']
            text = item['text']
            posts.append({'post_id': post_id, 'text': text})
        offset += 100
 
    return posts
 
 
def save_to_csv(posts):
    with open('vk_posts.csv', 'w', newline='', encoding='utf-8') as file:
        writer = csv.writer(file,delimiter=';')
        writer.writerow(['post_id', 'text'])
 
        for post in posts:
            writer.writerow([post['post_id'], post['text']])
 
 
if __name__ == "__main__":
    group_url = 'abitur_psu_mipt'
    posts = get_posts(group_url)
    save_to_csv(posts)

import pandas as pd
import pymorphy3
import nltk
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
 
nltk.download('punkt')
nltk.download('stopwords')
 
punctuation_marks = ['!', ',', '(', ')', ':', '-', '?', '.', '..', '...']
stop_words = stopwords.words("russian")
morph = pymorphy3.MorphAnalyzer()
 
def preprocess(text, stop_words, punctuation_marks, morph):
    tokens = word_tokenize(text.lower())
    preprocessed_text = []
    for token in tokens:
        if token not in punctuation_marks:
            lemma = morph.parse(token)[0].normal_form
            if lemma not in stop_words:
                preprocessed_text.append(lemma)
    return preprocessed_text
 
posts = pd.read_csv('vk_posts.csv', sep=";", names=['post_id', 'text'])
posts['preprocessed_text'] = posts['text'].apply(lambda post: preprocess(post, stop_words, punctuation_marks, morph))
print(posts)